From 653595a5fd94bb2d797ddec64d78b31a48215974 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 17 Jul 2018 11:03:13 +0100 Subject: [PATCH 1/7] Add NodeName to the alloc/job status outputs. Currently when operators need to log onto a machine where an alloc is running they will need to perform both an alloc/job status call and then a call to discover the node name from the node list. This updates both the job status and alloc status output to include the node name within the information to make operator use easier. Closes #2359 Cloess #1180 --- api/allocations.go | 2 ++ command/alloc_status.go | 1 + command/job_status.go | 10 ++++++---- nomad/structs/structs.go | 5 +++++ scheduler/generic_sched.go | 2 +- scheduler/system_sched.go | 2 +- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/api/allocations.go b/api/allocations.go index d622f86b0a5..80987fb4b45 100644 --- a/api/allocations.go +++ b/api/allocations.go @@ -86,6 +86,7 @@ type Allocation struct { EvalID string Name string NodeID string + NodeName string JobID string Job *Job TaskGroup string @@ -149,6 +150,7 @@ type AllocationListStub struct { Name string Namespace string NodeID string + NodeName string JobID string JobType string JobVersion uint64 diff --git a/command/alloc_status.go b/command/alloc_status.go index 44ae2bb0a10..b656657b8aa 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -233,6 +233,7 @@ func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, uuidLength)), fmt.Sprintf("Name|%s", alloc.Name), fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, uuidLength)), + fmt.Sprintf("Node Name|%s", alloc.NodeName), fmt.Sprintf("Job ID|%s", alloc.JobID), fmt.Sprintf("Job Version|%d", getVersion(alloc.Job)), fmt.Sprintf("Client Status|%s", alloc.ClientStatus), diff --git a/command/job_status.go b/command/job_status.go index 6ee4cd00abc..fb8336e2acf 100644 --- a/command/job_status.go +++ b/command/job_status.go @@ -413,12 +413,13 @@ func formatAllocListStubs(stubs []*api.AllocationListStub, verbose bool, uuidLen allocs := make([]string, len(stubs)+1) if verbose { - allocs[0] = "ID|Eval ID|Node ID|Task Group|Version|Desired|Status|Created|Modified" + allocs[0] = "ID|Eval ID|Node ID|Node Name|Task Group|Version|Desired|Status|Created|Modified" for i, alloc := range stubs { - allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s|%s", + allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%d|%s|%s|%s|%s", limit(alloc.ID, uuidLength), limit(alloc.EvalID, uuidLength), limit(alloc.NodeID, uuidLength), + alloc.NodeName, alloc.TaskGroup, alloc.JobVersion, alloc.DesiredStatus, @@ -427,14 +428,15 @@ func formatAllocListStubs(stubs []*api.AllocationListStub, verbose bool, uuidLen formatUnixNanoTime(alloc.ModifyTime)) } } else { - allocs[0] = "ID|Node ID|Task Group|Version|Desired|Status|Created|Modified" + allocs[0] = "ID|Node ID|Node Name|Task Group|Version|Desired|Status|Created|Modified" for i, alloc := range stubs { now := time.Now() createTimePretty := prettyTimeDiff(time.Unix(0, alloc.CreateTime), now) modTimePretty := prettyTimeDiff(time.Unix(0, alloc.ModifyTime), now) - allocs[i+1] = fmt.Sprintf("%s|%s|%s|%d|%s|%s|%s|%s", + allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s|%s", limit(alloc.ID, uuidLength), limit(alloc.NodeID, uuidLength), + alloc.NodeName, alloc.TaskGroup, alloc.JobVersion, alloc.DesiredStatus, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 88b2db3727c..514da5ae234 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -7154,6 +7154,9 @@ type Allocation struct { // NodeID is the node this is being placed on NodeID string + // NodeName is the name of the node this is being placed on. + NodeName string + // Job is the parent job of the task group being allocated. // This is copied at allocation time to avoid issues if the job // definition is updated. @@ -7615,6 +7618,7 @@ func (a *Allocation) Stub() *AllocListStub { Name: a.Name, Namespace: a.Namespace, NodeID: a.NodeID, + NodeName: a.NodeName, JobID: a.JobID, JobType: a.Job.Type, JobVersion: a.Job.Version, @@ -7642,6 +7646,7 @@ type AllocListStub struct { Name string Namespace string NodeID string + NodeName string JobID string JobType string JobVersion uint64 diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index 03b35b943ae..a9d6267a8a7 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -495,12 +495,12 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul TaskGroup: tg.Name, Metrics: s.ctx.Metrics(), NodeID: option.Node.ID, + NodeName: option.Node.Name, DeploymentID: deploymentID, TaskResources: resources.OldTaskResources(), AllocatedResources: resources, DesiredStatus: structs.AllocDesiredStatusRun, ClientStatus: structs.AllocClientStatusPending, - SharedResources: &structs.Resources{ DiskMB: tg.EphemeralDisk.SizeMB, }, diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index e3f4015fbd1..e00cea147ae 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -331,11 +331,11 @@ func (s *SystemScheduler) computePlacements(place []allocTuple) error { TaskGroup: missing.TaskGroup.Name, Metrics: s.ctx.Metrics(), NodeID: option.Node.ID, + NodeName: option.Node.Name, TaskResources: resources.OldTaskResources(), AllocatedResources: resources, DesiredStatus: structs.AllocDesiredStatusRun, ClientStatus: structs.AllocClientStatusPending, - SharedResources: &structs.Resources{ DiskMB: missing.TaskGroup.EphemeralDisk.SizeMB, }, From 2193d1aecd40f44d6df134a9ffe837c51e1e9e4a Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Fri, 18 Jan 2019 03:55:17 -0800 Subject: [PATCH 2/7] Don't display node name if output isn't verbose. Add tests. --- command/alloc_status_test.go | 12 ++++++++++++ command/job_status.go | 5 ++--- command/job_status_test.go | 21 +++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/command/alloc_status_test.go b/command/alloc_status_test.go index fbd19942497..e4b5c239c17 100644 --- a/command/alloc_status_test.go +++ b/command/alloc_status_test.go @@ -120,9 +120,11 @@ func TestAllocStatusCommand_Run(t *testing.T) { } // get an alloc id allocId1 := "" + nodeName := "" if allocs, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil { if len(allocs) > 0 { allocId1 = allocs[0].ID + nodeName = allocs[0].NodeName } } if allocId1 == "" { @@ -141,6 +143,16 @@ func TestAllocStatusCommand_Run(t *testing.T) { t.Fatalf("expected to have 'Modified' but saw: %s", out) } + if !strings.Contains(out, "Modified") { + t.Fatalf("expected to have 'Modified' but saw: %s", out) + } + + nodeNameRegexpStr := fmt.Sprintf(`\nNode Name\s+= %s\n`, regexp.QuoteMeta(nodeName)) + nodeNameRegexp := regexp.MustCompile(nodeNameRegexpStr) + if !nodeNameRegexp.MatchString(out) { + t.Fatalf("expected to have 'Node Name' but saw: %s", out) + } + ui.OutputWriter.Reset() if code := cmd.Run([]string{"-address=" + url, "-verbose", allocId1}); code != 0 { diff --git a/command/job_status.go b/command/job_status.go index fb8336e2acf..63fd163e208 100644 --- a/command/job_status.go +++ b/command/job_status.go @@ -428,15 +428,14 @@ func formatAllocListStubs(stubs []*api.AllocationListStub, verbose bool, uuidLen formatUnixNanoTime(alloc.ModifyTime)) } } else { - allocs[0] = "ID|Node ID|Node Name|Task Group|Version|Desired|Status|Created|Modified" + allocs[0] = "ID|Node ID|Task Group|Version|Desired|Status|Created|Modified" for i, alloc := range stubs { now := time.Now() createTimePretty := prettyTimeDiff(time.Unix(0, alloc.CreateTime), now) modTimePretty := prettyTimeDiff(time.Unix(0, alloc.ModifyTime), now) - allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s|%s", + allocs[i+1] = fmt.Sprintf("%s|%s|%s|%d|%s|%s|%s|%s", limit(alloc.ID, uuidLength), limit(alloc.NodeID, uuidLength), - alloc.NodeName, alloc.TaskGroup, alloc.JobVersion, alloc.DesiredStatus, diff --git a/command/job_status_test.go b/command/job_status_test.go index e0bc40b565b..f4a71c7b8c9 100644 --- a/command/job_status_test.go +++ b/command/job_status_test.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "regexp" "strings" "testing" "time" @@ -123,6 +124,14 @@ func TestJobStatusCommand_Run(t *testing.T) { if code := cmd.Run([]string{"-address=" + url, "-verbose", "job2_sfx"}); code != 0 { t.Fatalf("expected exit 0, got: %d", code) } + + nodeName := "" + if allocs, _, err := client.Jobs().Allocations("job2_sfx", false, nil); err == nil { + if len(allocs) > 0 { + nodeName = allocs[0].NodeName + } + } + out = ui.OutputWriter.String() if strings.Contains(out, "job1_sfx") || !strings.Contains(out, "job2_sfx") { t.Fatalf("expected only job2_sfx, got: %s", out) @@ -139,6 +148,18 @@ func TestJobStatusCommand_Run(t *testing.T) { if !strings.Contains(out, "Modified") { t.Fatal("should have modified header") } + + // string calculations based on 1-byte chars, not using runes + allocationsTableName := "Allocations\n" + allocationsTableStr := strings.Split(out, allocationsTableName)[1] + nodeNameHeaderStr := "Node Name" + nodeNameHeaderIndex := strings.Index(allocationsTableStr, nodeNameHeaderStr) + nodeNameRegexpStr := fmt.Sprintf(`.*%s.*\n.{%d}%s`, nodeNameHeaderStr, nodeNameHeaderIndex, regexp.QuoteMeta(nodeName)) + nodeNameRegexp := regexp.MustCompile(nodeNameRegexpStr) + if !nodeNameRegexp.MatchString(out) { + t.Fatalf("expected to have 'Node Name' but saw: %s", out) + } + ui.ErrorWriter.Reset() ui.OutputWriter.Reset() From a2130bcf7d8e9b6bbbac5c229ca92ea901e58585 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Wed, 23 Jan 2019 09:33:22 -0800 Subject: [PATCH 3/7] Remove redundant assertion and replace regex matches with require --- command/alloc_status_test.go | 9 +-------- command/job_status_test.go | 5 +---- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/command/alloc_status_test.go b/command/alloc_status_test.go index e4b5c239c17..b3c7fef2070 100644 --- a/command/alloc_status_test.go +++ b/command/alloc_status_test.go @@ -143,15 +143,8 @@ func TestAllocStatusCommand_Run(t *testing.T) { t.Fatalf("expected to have 'Modified' but saw: %s", out) } - if !strings.Contains(out, "Modified") { - t.Fatalf("expected to have 'Modified' but saw: %s", out) - } - nodeNameRegexpStr := fmt.Sprintf(`\nNode Name\s+= %s\n`, regexp.QuoteMeta(nodeName)) - nodeNameRegexp := regexp.MustCompile(nodeNameRegexpStr) - if !nodeNameRegexp.MatchString(out) { - t.Fatalf("expected to have 'Node Name' but saw: %s", out) - } + require.Regexp(t, regexp.MustCompile(nodeNameRegexpStr), out) ui.OutputWriter.Reset() diff --git a/command/job_status_test.go b/command/job_status_test.go index f4a71c7b8c9..745af018e4c 100644 --- a/command/job_status_test.go +++ b/command/job_status_test.go @@ -155,10 +155,7 @@ func TestJobStatusCommand_Run(t *testing.T) { nodeNameHeaderStr := "Node Name" nodeNameHeaderIndex := strings.Index(allocationsTableStr, nodeNameHeaderStr) nodeNameRegexpStr := fmt.Sprintf(`.*%s.*\n.{%d}%s`, nodeNameHeaderStr, nodeNameHeaderIndex, regexp.QuoteMeta(nodeName)) - nodeNameRegexp := regexp.MustCompile(nodeNameRegexpStr) - if !nodeNameRegexp.MatchString(out) { - t.Fatalf("expected to have 'Node Name' but saw: %s", out) - } + require.Regexp(t, regexp.MustCompile(nodeNameRegexpStr), out) ui.ErrorWriter.Reset() ui.OutputWriter.Reset() From 5ca664931b0a8fab04851a5351e485eae31be29d Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Mon, 4 Mar 2019 01:49:32 -0800 Subject: [PATCH 4/7] Add code for plan normalization --- nomad/fsm.go | 9 +- nomad/job_endpoint.go | 3 +- nomad/leader.go | 4 +- nomad/operator_endpoint.go | 4 +- nomad/plan_apply.go | 139 +++++++++++++++++++++--------- nomad/server.go | 4 +- nomad/state/state_store.go | 172 ++++++++++++++++++++++++++----------- nomad/structs/structs.go | 71 ++++++++++++--- nomad/util.go | 9 +- nomad/util_test.go | 2 +- nomad/worker.go | 6 +- nomad/worker_test.go | 2 +- scheduler/generic_sched.go | 32 ++++--- scheduler/scheduler.go | 6 +- scheduler/system_sched.go | 23 +++-- scheduler/testing.go | 2 +- scheduler/util.go | 9 +- 17 files changed, 349 insertions(+), 148 deletions(-) diff --git a/nomad/fsm.go b/nomad/fsm.go index 3c91b8f5c69..fc3cebad1b4 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -86,6 +86,9 @@ type nomadFSM struct { // new state store). Everything internal here is synchronized by the // Raft side, so doesn't need to lock this. stateLock sync.RWMutex + + // Reference to the server the FSM is running on + server *Server } // nomadSnapshot is used to provide a snapshot of the current @@ -121,7 +124,7 @@ type FSMConfig struct { } // NewFSMPath is used to construct a new FSM with a blank state -func NewFSM(config *FSMConfig) (*nomadFSM, error) { +func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { // Create a state store sconfig := &state.StateStoreConfig{ Logger: config.Logger, @@ -142,6 +145,7 @@ func NewFSM(config *FSMConfig) (*nomadFSM, error) { timetable: NewTimeTable(timeTableGranularity, timeTableLimit), enterpriseAppliers: make(map[structs.MessageType]LogApplier, 8), enterpriseRestorers: make(map[SnapshotType]SnapshotRestorer, 8), + server: server, } // Register all the log applier functions @@ -1423,7 +1427,8 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner) + allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanDenormalization, true) + sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) if err != nil { return err } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 132c14f58ce..f6a0ace535c 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1213,7 +1213,8 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner) + allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanDenormalization, true) + sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) if err != nil { return err } diff --git a/nomad/leader.go b/nomad/leader.go index 8e26a56f514..17c8199962f 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -1243,7 +1243,7 @@ func (s *Server) getOrCreateAutopilotConfig() *structs.AutopilotConfig { return config } - if !ServersMeetMinimumVersion(s.Members(), minAutopilotVersion) { + if !ServersMeetMinimumVersion(s.Members(), minAutopilotVersion, false) { s.logger.Named("autopilot").Warn("can't initialize until all servers are above minimum version", "min_version", minAutopilotVersion) return nil } @@ -1270,7 +1270,7 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration { if config != nil { return config } - if !ServersMeetMinimumVersion(s.Members(), minSchedulerConfigVersion) { + if !ServersMeetMinimumVersion(s.Members(), minSchedulerConfigVersion, false) { s.logger.Named("core").Warn("can't initialize scheduler config until all servers are above minimum version", "min_version", minSchedulerConfigVersion) return nil } diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go index fc9edabdd97..c44e14c31c3 100644 --- a/nomad/operator_endpoint.go +++ b/nomad/operator_endpoint.go @@ -237,7 +237,7 @@ func (op *Operator) AutopilotSetConfiguration(args *structs.AutopilotSetConfigRe } // All servers should be at or above 0.8.0 to apply this operatation - if !ServersMeetMinimumVersion(op.srv.Members(), minAutopilotVersion) { + if !ServersMeetMinimumVersion(op.srv.Members(), minAutopilotVersion, false) { return fmt.Errorf("All servers should be running version %v to update autopilot config", minAutopilotVersion) } @@ -305,7 +305,7 @@ func (op *Operator) SchedulerSetConfiguration(args *structs.SchedulerSetConfigRe } // All servers should be at or above 0.9.0 to apply this operatation - if !ServersMeetMinimumVersion(op.srv.Members(), minSchedulerConfigVersion) { + if !ServersMeetMinimumVersion(op.srv.Members(), minSchedulerConfigVersion, false) { return fmt.Errorf("All servers should be running version %v to update scheduler config", minSchedulerConfigVersion) } // Apply the update diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index f40690d00f2..dc991323288 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -15,7 +15,7 @@ import ( "github.com/hashicorp/raft" ) -// planner is used to mange the submitted allocation plans that are waiting +// planner is used to manage the submitted allocation plans that are waiting // to be accessed by the leader type planner struct { *Server @@ -149,52 +149,90 @@ func (p *planner) planApply() { // applyPlan is used to apply the plan result and to return the alloc index func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap *state.StateSnapshot) (raft.ApplyFuture, error) { - // Determine the minimum number of updates, could be more if there - // are multiple updates per node - minUpdates := len(result.NodeUpdate) - minUpdates += len(result.NodeAllocation) - // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: plan.Job, - Alloc: make([]*structs.Allocation, 0, minUpdates), + Job: plan.Job, }, Deployment: result.Deployment, DeploymentUpdates: result.DeploymentUpdates, EvalID: plan.EvalID, NodePreemptions: make([]*structs.Allocation, 0, len(result.NodePreemptions)), } - for _, updateList := range result.NodeUpdate { - req.Alloc = append(req.Alloc, updateList...) - } - for _, allocList := range result.NodeAllocation { - req.Alloc = append(req.Alloc, allocList...) - } - for _, preemptions := range result.NodePreemptions { - req.NodePreemptions = append(req.NodePreemptions, preemptions...) - } - - // Set the time the alloc was applied for the first time. This can be used - // to approximate the scheduling time. + preemptedJobIDs := make(map[structs.NamespacedID]struct{}) now := time.Now().UTC().UnixNano() - for _, alloc := range req.Alloc { - if alloc.CreateTime == 0 { - alloc.CreateTime = now + + if ServersMeetMinimumVersion(p.Members(), MinVersionPlanDenormalization, true) { + // Initialize the allocs request using the new optimized log entry format. + // Determine the minimum number of updates, could be more if there + // are multiple updates per node + req.AllocsStopped = make([]*structs.Allocation, 0, len(result.NodeUpdate)) + req.AllocsUpdated = make([]*structs.Allocation, 0, len(result.NodeAllocation)) + + for _, updateList := range result.NodeUpdate { + for _, stoppedAlloc := range updateList { + req.AllocsStopped = append(req.AllocsStopped, &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: stoppedAlloc.DesiredDescription, + ClientStatus: stoppedAlloc.ClientStatus, + ModifyTime: now, + }) + } } - alloc.ModifyTime = now - } - // Set modify time for preempted allocs if any - // Also gather jobids to create follow up evals - preemptedJobIDs := make(map[structs.NamespacedID]struct{}) - for _, alloc := range req.NodePreemptions { - alloc.ModifyTime = now - id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} - _, ok := preemptedJobIDs[id] - if !ok { - preemptedJobIDs[id] = struct{}{} + for _, allocList := range result.NodeAllocation { + req.AllocsUpdated = append(req.AllocsUpdated, allocList...) + } + + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + updateAllocTimestamps(req.AllocsUpdated, now) + + for _, preemptions := range result.NodePreemptions { + for _, preemptedAlloc := range preemptions { + req.NodePreemptions = append(req.NodePreemptions, &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, + ModifyTime: now, + }) + + // Gather jobids to create follow up evals + appendNamespacedJobID(preemptedJobIDs, preemptedAlloc) + } + } + } else { + // Deprecated: This code path is deprecated and will only be used to support + // application of older log entries. Expected to be removed in a future version. + + // Determine the minimum number of updates, could be more if there + // are multiple updates per node + minUpdates := len(result.NodeUpdate) + minUpdates += len(result.NodeAllocation) + + // Initialize the allocs request using the older log entry format + req.Alloc = make([]*structs.Allocation, 0, minUpdates) + + for _, updateList := range result.NodeUpdate { + req.Alloc = append(req.Alloc, updateList...) + } + for _, allocList := range result.NodeAllocation { + req.Alloc = append(req.Alloc, allocList...) + } + + for _, preemptions := range result.NodePreemptions { + req.NodePreemptions = append(req.NodePreemptions, preemptions...) + } + + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + updateAllocTimestamps(req.Alloc, now) + + // Set modify time for preempted allocs if any + // Also gather jobids to create follow up evals + for _, alloc := range req.NodePreemptions { + alloc.ModifyTime = now + appendNamespacedJobID(preemptedJobIDs, alloc) } } @@ -232,6 +270,22 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap return future, nil } +func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *structs.Allocation) { + id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} + if _, ok := jobIDs[id]; !ok { + jobIDs[id] = struct{}{} + } +} + +func updateAllocTimestamps(allocations []*structs.Allocation, timestamp int64) { + for _, alloc := range allocations { + if alloc.CreateTime == 0 { + alloc.CreateTime = timestamp + } + alloc.ModifyTime = timestamp + } +} + // asyncPlanWait is used to apply and respond to a plan async func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, result *structs.PlanResult, pending *pendingPlan) { @@ -264,6 +318,15 @@ func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, func evaluatePlan(pool *EvaluatePool, snap *state.StateSnapshot, plan *structs.Plan, logger log.Logger) (*structs.PlanResult, error) { defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now()) + err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, plan.Job) + if err != nil { + return nil, err + } + err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, plan.Job) + if err != nil { + return nil, err + } + // Check if the plan exceeds quota overQuota, err := evaluatePlanQuota(snap, plan) if err != nil { @@ -521,15 +584,11 @@ func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID stri // Remove any preempted allocs if preempted := plan.NodePreemptions[nodeID]; len(preempted) > 0 { - for _, allocs := range preempted { - remove = append(remove, allocs) - } + remove = append(remove, preempted...) } if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 { - for _, alloc := range updated { - remove = append(remove, alloc) - } + remove = append(remove, updated...) } proposed := structs.RemoveAllocs(existingAlloc, remove) proposed = append(proposed, plan.NodeAllocation[nodeID]...) diff --git a/nomad/server.go b/nomad/server.go index 89da7a9e4b9..23142580d96 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1078,7 +1078,7 @@ func (s *Server) setupRaft() error { Region: s.Region(), } var err error - s.fsm, err = NewFSM(fsmConfig) + s.fsm, err = NewFSM(fsmConfig, s) if err != nil { return err } @@ -1173,7 +1173,7 @@ func (s *Server) setupRaft() error { if err != nil { return fmt.Errorf("recovery failed to parse peers.json: %v", err) } - tmpFsm, err := NewFSM(fsmConfig) + tmpFsm, err := NewFSM(fsmConfig, s) if err != nil { return fmt.Errorf("recovery failed to make temp FSM: %v", err) } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 84201cbaaf0..24b90a2f45b 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -170,6 +170,21 @@ RUN_QUERY: // UpsertPlanResults is used to upsert the results of a plan. func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanResultsRequest) error { + snapshot, err := s.Snapshot() + if err != nil { + return err + } + + err = snapshot.DenormalizeAllocationsSlice(results.AllocsStopped, results.Job) + if err != nil { + return err + } + + err = snapshot.DenormalizeAllocationsSlice(results.NodePreemptions, results.Job) + if err != nil { + return err + } + txn := s.db.Txn(true) defer txn.Abort() @@ -185,34 +200,6 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR s.upsertDeploymentUpdates(index, results.DeploymentUpdates, txn) } - // Attach the job to all the allocations. It is pulled out in the payload to - // avoid the redundancy of encoding, but should be denormalized prior to - // being inserted into MemDB. - structs.DenormalizeAllocationJobs(results.Job, results.Alloc) - - // COMPAT(0.11): Remove in 0.11 - // Calculate the total resources of allocations. It is pulled out in the - // payload to avoid encoding something that can be computed, but should be - // denormalized prior to being inserted into MemDB. - for _, alloc := range results.Alloc { - if alloc.Resources != nil { - continue - } - - alloc.Resources = new(structs.Resources) - for _, task := range alloc.TaskResources { - alloc.Resources.Add(task) - } - - // Add the shared resources - alloc.Resources.Add(alloc.SharedResources) - } - - // Upsert the allocations - if err := s.upsertAllocsImpl(index, results.Alloc, txn); err != nil { - return err - } - // COMPAT: Nomad versions before 0.7.1 did not include the eval ID when // applying the plan. Thus while we are upgrading, we ignore updating the // modify index of evaluations from older plans. @@ -223,35 +210,33 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR } } - // Prepare preempted allocs in the plan results for update - var preemptedAllocs []*structs.Allocation - for _, preemptedAlloc := range results.NodePreemptions { - // Look for existing alloc - existing, err := txn.First("allocs", "id", preemptedAlloc.ID) - if err != nil { - return fmt.Errorf("alloc lookup failed: %v", err) - } + noOfAllocs := len(results.NodePreemptions) - // Nothing to do if this does not exist - if existing == nil { - continue - } - exist := existing.(*structs.Allocation) + if len(results.Alloc) > 0 { + // COMPAT 0.11: This branch will be removed, when Alloc is removed + // Attach the job to all the allocations. It is pulled out in the payload to + // avoid the redundancy of encoding, but should be denormalized prior to + // being inserted into MemDB. + addComputedAllocAttrs(results.Alloc, results.Job) + noOfAllocs += len(results.Alloc) + } else { + // Attach the job to all the allocations. It is pulled out in the payload to + // avoid the redundancy of encoding, but should be denormalized prior to + // being inserted into MemDB. + addComputedAllocAttrs(results.AllocsUpdated, results.Job) + noOfAllocs += len(results.AllocsStopped) + len(results.AllocsUpdated) + } - // Copy everything from the existing allocation - copyAlloc := exist.Copy() + allocsToUpsert := make([]*structs.Allocation, 0, noOfAllocs) - // Only update the fields set by the scheduler - copyAlloc.DesiredStatus = preemptedAlloc.DesiredStatus - copyAlloc.PreemptedByAllocation = preemptedAlloc.PreemptedByAllocation - copyAlloc.DesiredDescription = preemptedAlloc.DesiredDescription - copyAlloc.ModifyTime = preemptedAlloc.ModifyTime - preemptedAllocs = append(preemptedAllocs, copyAlloc) + // COMPAT 0.11: This append should be removed when Alloc is removed + allocsToUpsert = append(allocsToUpsert, results.Alloc...) - } + allocsToUpsert = append(allocsToUpsert, results.AllocsStopped...) + allocsToUpsert = append(allocsToUpsert, results.AllocsUpdated...) + allocsToUpsert = append(allocsToUpsert, results.NodePreemptions...) - // Upsert the preempted allocations - if err := s.upsertAllocsImpl(index, preemptedAllocs, txn); err != nil { + if err := s.upsertAllocsImpl(index, allocsToUpsert, txn); err != nil { return err } @@ -266,6 +251,28 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR return nil } +func addComputedAllocAttrs(allocs []*structs.Allocation, job *structs.Job) { + structs.DenormalizeAllocationJobs(job, allocs) + + // COMPAT(0.11): Remove in 0.11 + // Calculate the total resources of allocations. It is pulled out in the + // payload to avoid encoding something that can be computed, but should be + // denormalized prior to being inserted into MemDB. + for _, alloc := range allocs { + if alloc.Resources != nil { + continue + } + + alloc.Resources = new(structs.Resources) + for _, task := range alloc.TaskResources { + alloc.Resources.Add(task) + } + + // Add the shared resources + alloc.Resources.Add(alloc.SharedResources) + } +} + // upsertDeploymentUpdates updates the deployments given the passed status // updates. func (s *StateStore) upsertDeploymentUpdates(index uint64, updates []*structs.DeploymentStatusUpdate, txn *memdb.Txn) error { @@ -4100,6 +4107,67 @@ type StateSnapshot struct { StateStore } +// DenormalizeAllocationsMap takes in a map of nodes to allocations, and queries the +// Allocation for each of the Allocation diffs and merges the updated attributes with +// the existing Allocation, and attaches the Job provided +func (s *StateSnapshot) DenormalizeAllocationsMap(nodeAllocations map[string][]*structs.Allocation, job *structs.Job) error { + for _, allocDiffs := range nodeAllocations { + if err := s.DenormalizeAllocationsSlice(allocDiffs, job); err != nil { + return err + } + } + return nil +} + +// DenormalizeAllocationsSlice queries the Allocation for each of the Allocation diffs and merges +// the updated attributes with the existing Allocation, and attaches the Job provided +func (s *StateSnapshot) DenormalizeAllocationsSlice(allocDiffs []*structs.Allocation, job *structs.Job) error { + // Output index for denormalized Allocations + j := 0 + + for _, allocDiff := range allocDiffs { + alloc, err := s.AllocByID(nil, allocDiff.ID) + if err != nil { + return fmt.Errorf("alloc lookup failed: %v", err) + } + if alloc == nil { + continue + } + + // Merge the updates to the Allocation + allocCopy := alloc.CopySkipJob() + allocCopy.Job = job + + if allocDiff.PreemptedByAllocation != "" { + // If alloc is a preemption + allocCopy.PreemptedByAllocation = allocDiff.PreemptedByAllocation + allocCopy.DesiredDescription = getPreemptedAllocDesiredDescription(allocDiff.PreemptedByAllocation) + allocCopy.DesiredStatus = structs.AllocDesiredStatusEvict + } else { + // If alloc is a stopped alloc + allocCopy.DesiredDescription = allocDiff.DesiredDescription + allocCopy.DesiredStatus = structs.AllocDesiredStatusStop + if allocDiff.ClientStatus != "" { + allocCopy.ClientStatus = allocDiff.ClientStatus + } + } + if allocDiff.ModifyTime != 0 { + allocCopy.ModifyTime = allocDiff.ModifyTime + } + + // Update the allocDiff in the slice to equal the denormalized alloc + allocDiffs[j] = allocCopy + j++ + } + // Retain only the denormalized Allocations in the slice + allocDiffs = allocDiffs[:j] + return nil +} + +func getPreemptedAllocDesiredDescription(PreemptedByAllocID string) string { + return fmt.Sprintf("Preempted by alloc ID %v", PreemptedByAllocID) +} + // StateRestore is used to optimize the performance when // restoring state by only using a single large transaction // instead of thousands of sub transactions diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 514da5ae234..be089165a74 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -28,8 +28,8 @@ import ( "github.com/gorhill/cronexpr" "github.com/hashicorp/consul/api" hcodec "github.com/hashicorp/go-msgpack/codec" - multierror "github.com/hashicorp/go-multierror" - version "github.com/hashicorp/go-version" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/args" @@ -654,9 +654,9 @@ type ApplyPlanResultsRequest struct { // the evaluation itself being updated. EvalID string - // NodePreemptions is a slice of allocations from other lower priority jobs + // NodePreemptions is a slice of allocation diffs from other lower priority jobs // that are preempted. Preempted allocations are marked as evicted. - NodePreemptions []*Allocation + NodePreemptions []*AllocationDiff // PreemptionEvals is a slice of follow up evals for jobs whose allocations // have been preempted to place allocs in this plan @@ -668,8 +668,16 @@ type ApplyPlanResultsRequest struct { // within a single transaction type AllocUpdateRequest struct { // Alloc is the list of new allocations to assign + // Deprecated: Replaced with two separate slices, one containing stopped allocations + // and another containing updated allocations Alloc []*Allocation + // Allocations to stop. Contains only the diff, not the entire allocation + AllocsStopped []*AllocationDiff + + // New or updated allocations + AllocsUpdated []*Allocation + // Evals is the list of new evaluations to create // Evals are valid only when used in the Raft RPC Evals []*Evaluation @@ -7139,6 +7147,9 @@ const ( // Allocation is used to allocate the placement of a task group to a node. type Allocation struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // ID of the allocation (UUID) ID string @@ -7253,6 +7264,10 @@ type Allocation struct { ModifyTime int64 } +// AllocationDiff is a type alias for Allocation used to indicate that a diff is +// and not the entire allocation +type AllocationDiff = Allocation + // Index returns the index of the allocation. If the allocation is from a task // group with count greater than 1, there will be multiple allocations for it. func (a *Allocation) Index() uint { @@ -7267,11 +7282,12 @@ func (a *Allocation) Index() uint { return uint(num) } +// Copy provides a copy of the allocation and deep copies the job func (a *Allocation) Copy() *Allocation { return a.copyImpl(true) } -// Copy provides a copy of the allocation but doesn't deep copy the job +// CopySkipJob provides a copy of the allocation but doesn't deep copy the job func (a *Allocation) CopySkipJob() *Allocation { return a.copyImpl(false) } @@ -8003,6 +8019,9 @@ const ( // potentially taking action (allocation of work) or doing nothing if the state // of the world does not require it. type Evaluation struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // ID is a randomly generated UUID used for this evaluation. This // is assigned upon the creation of the evaluation. ID string @@ -8193,7 +8212,7 @@ func (e *Evaluation) ShouldBlock() bool { // MakePlan is used to make a plan from the given evaluation // for a given Job -func (e *Evaluation) MakePlan(j *Job) *Plan { +func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { p := &Plan{ EvalID: e.ID, Priority: e.Priority, @@ -8201,6 +8220,7 @@ func (e *Evaluation) MakePlan(j *Job) *Plan { NodeUpdate: make(map[string][]*Allocation), NodeAllocation: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), + NormalizeAllocs: allowPlanOptimization, } if j != nil { p.AllAtOnce = j.AllAtOnce @@ -8270,6 +8290,9 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { // are submitted to the leader which verifies that resources have // not been overcommitted before admitting the plan. type Plan struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // EvalID is the evaluation ID this plan is associated with EvalID string @@ -8319,11 +8342,14 @@ type Plan struct { // lower priority jobs that are preempted. Preempted allocations are marked // as evicted. NodePreemptions map[string][]*Allocation + + // Indicates whether allocs are normalized in the Plan + NormalizeAllocs bool `codec:"-"` } -// AppendUpdate marks the allocation for eviction. The clientStatus of the +// AppendStoppedAlloc marks an allocation to be stopped. The clientStatus of the // allocation may be optionally set by passing in a non-empty value. -func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clientStatus string) { +func (p *Plan) AppendStoppedAlloc(alloc *Allocation, desiredDesc, clientStatus string) { newAlloc := new(Allocation) *newAlloc = *alloc @@ -8339,7 +8365,7 @@ func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clien // Strip the resources as it can be rebuilt. newAlloc.Resources = nil - newAlloc.DesiredStatus = desiredStatus + newAlloc.DesiredStatus = AllocDesiredStatusStop newAlloc.DesiredDescription = desiredDesc if clientStatus != "" { @@ -8353,12 +8379,12 @@ func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clien // AppendPreemptedAlloc is used to append an allocation that's being preempted to the plan. // To minimize the size of the plan, this only sets a minimal set of fields in the allocation -func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, desiredStatus, preemptingAllocID string) { +func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, preemptingAllocID string) { newAlloc := &Allocation{} newAlloc.ID = alloc.ID newAlloc.JobID = alloc.JobID newAlloc.Namespace = alloc.Namespace - newAlloc.DesiredStatus = desiredStatus + newAlloc.DesiredStatus = AllocDesiredStatusEvict newAlloc.PreemptedByAllocation = preemptingAllocID desiredDesc := fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID) @@ -8411,6 +8437,29 @@ func (p *Plan) IsNoOp() bool { len(p.DeploymentUpdates) == 0 } +func (p *Plan) NormalizeAllocations() { + if p.NormalizeAllocs { + for _, allocs := range p.NodeUpdate { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + DesiredDescription: alloc.DesiredDescription, + ClientStatus: alloc.ClientStatus, + } + } + } + + for _, allocs := range p.NodePreemptions { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: alloc.PreemptedByAllocation, + } + } + } + } +} + // PlanResult is the result of a plan submitted to the leader. type PlanResult struct { // NodeUpdate contains all the updates that were committed. diff --git a/nomad/util.go b/nomad/util.go index 44b7119242b..8a6ff407f73 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -14,6 +14,11 @@ import ( "github.com/hashicorp/serf/serf" ) +// MinVersionPlanDenormalization is the minimum version to support the +// denormalization of Plan in SubmitPlan, and the raft log entry committed +// in ApplyPlanResultsRequest +var MinVersionPlanDenormalization = version.Must(version.NewVersion("0.9.1")) + // ensurePath is used to make sure a path exists func ensurePath(path string, dir bool) error { if !dir { @@ -145,9 +150,9 @@ func isNomadServer(m serf.Member) (bool, *serverParts) { // ServersMeetMinimumVersion returns whether the given alive servers are at least on the // given Nomad version -func ServersMeetMinimumVersion(members []serf.Member, minVersion *version.Version) bool { +func ServersMeetMinimumVersion(members []serf.Member, minVersion *version.Version, checkFailedServers bool) bool { for _, member := range members { - if valid, parts := isNomadServer(member); valid && parts.Status == serf.StatusAlive { + if valid, parts := isNomadServer(member); valid && (parts.Status == serf.StatusAlive || (checkFailedServers && parts.Status == serf.StatusFailed)) { // Check if the versions match - version.LessThan will return true for // 0.8.0-rc1 < 0.8.0, so we want to ignore the metadata versionsMatch := slicesMatch(minVersion.Segments(), parts.Build.Segments()) diff --git a/nomad/util_test.go b/nomad/util_test.go index 4fe670cdc71..5614067609c 100644 --- a/nomad/util_test.go +++ b/nomad/util_test.go @@ -162,7 +162,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { } for _, tc := range cases { - result := ServersMeetMinimumVersion(tc.members, tc.ver) + result := ServersMeetMinimumVersion(tc.members, tc.ver, false) if result != tc.expected { t.Fatalf("bad: %v, %v, %v", result, tc.ver.String(), tc) } diff --git a/nomad/worker.go b/nomad/worker.go index f0aefb62d54..6cd8e0107fc 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -284,7 +284,8 @@ func (w *Worker) invokeScheduler(eval *structs.Evaluation, token string) error { if eval.Type == structs.JobTypeCore { sched = NewCoreScheduler(w.srv, snap) } else { - sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w) + allowPlanOptimization := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanDenormalization, true) + sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w, allowPlanOptimization) if err != nil { return fmt.Errorf("failed to instantiate scheduler: %v", err) } @@ -310,6 +311,9 @@ func (w *Worker) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, scheduler. // Add the evaluation token to the plan plan.EvalToken = w.evalToken + // Normalize stopped and preempted allocs before RPC + plan.NormalizeAllocations() + // Setup the request req := structs.PlanRequest{ Plan: plan, diff --git a/nomad/worker_test.go b/nomad/worker_test.go index 2f3e3172831..b7a9e526a21 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -36,7 +36,7 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { } func init() { - scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { + scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { n := &NoopScheduler{ state: s, planner: p, diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index a9d6267a8a7..ebdac6721cf 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -77,6 +77,10 @@ type GenericScheduler struct { planner Planner batch bool + // Temporary flag introduced till the code for sending/committing full allocs in the Plan can + // be safely removed + allowPlanOptimization bool + eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -94,23 +98,25 @@ type GenericScheduler struct { } // NewServiceScheduler is a factory function to instantiate a new service scheduler -func NewServiceScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewServiceScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { s := &GenericScheduler{ - logger: logger.Named("service_sched"), - state: state, - planner: planner, - batch: false, + logger: logger.Named("service_sched"), + state: state, + planner: planner, + batch: false, + allowPlanOptimization: allowPlanOptimization, } return s } // NewBatchScheduler is a factory function to instantiate a new batch scheduler -func NewBatchScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewBatchScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { s := &GenericScheduler{ - logger: logger.Named("batch_sched"), - state: state, - planner: planner, - batch: true, + logger: logger.Named("batch_sched"), + state: state, + planner: planner, + batch: true, + allowPlanOptimization: allowPlanOptimization, } return s } @@ -223,7 +229,7 @@ func (s *GenericScheduler) process() (bool, error) { s.followUpEvals = nil // Create a plan - s.plan = s.eval.MakePlan(s.job) + s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) if !s.batch { // Get any existing deployment @@ -365,7 +371,7 @@ func (s *GenericScheduler) computeJobAllocs() error { // Handle the stop for _, stop := range results.stop { - s.plan.AppendUpdate(stop.alloc, structs.AllocDesiredStatusStop, stop.statusDescription, stop.clientStatus) + s.plan.AppendStoppedAlloc(stop.alloc, stop.statusDescription, stop.clientStatus) } // Handle the in-place updates @@ -463,7 +469,7 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul stopPrevAlloc, stopPrevAllocDesc := missing.StopPreviousAlloc() prevAllocation := missing.PreviousAllocation() if stopPrevAlloc { - s.plan.AppendUpdate(prevAllocation, structs.AllocDesiredStatusStop, stopPrevAllocDesc, "") + s.plan.AppendStoppedAlloc(prevAllocation, stopPrevAllocDesc, "") } // Compute penalty nodes for rescheduled allocs diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 639b2b8cf28..76a0a69408c 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -28,7 +28,7 @@ var BuiltinSchedulers = map[string]Factory{ // NewScheduler is used to instantiate and return a new scheduler // given the scheduler name, initial state, and planner. -func NewScheduler(name string, logger log.Logger, state State, planner Planner) (Scheduler, error) { +func NewScheduler(name string, logger log.Logger, state State, planner Planner, allowPlanOptimization bool) (Scheduler, error) { // Lookup the factory function factory, ok := BuiltinSchedulers[name] if !ok { @@ -36,12 +36,12 @@ func NewScheduler(name string, logger log.Logger, state State, planner Planner) } // Instantiate the scheduler - sched := factory(logger, state, planner) + sched := factory(logger, state, planner, allowPlanOptimization) return sched, nil } // Factory is used to instantiate a new Scheduler -type Factory func(log.Logger, State, Planner) Scheduler +type Factory func(log.Logger, State, Planner, bool) Scheduler // Scheduler is the top level instance for a scheduler. A scheduler is // meant to only encapsulate business logic, pushing the various plumbing diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index e00cea147ae..59edb31abb5 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -23,6 +23,10 @@ type SystemScheduler struct { state State planner Planner + // Temporary flag introduced till the code for sending/committing full allocs in the Plan can + // be safely removed + allowPlanOptimization bool + eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -41,11 +45,12 @@ type SystemScheduler struct { // NewSystemScheduler is a factory function to instantiate a new system // scheduler. -func NewSystemScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewSystemScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { return &SystemScheduler{ - logger: logger.Named("system_sched"), - state: state, - planner: planner, + logger: logger.Named("system_sched"), + state: state, + planner: planner, + allowPlanOptimization: allowPlanOptimization, } } @@ -110,7 +115,7 @@ func (s *SystemScheduler) process() (bool, error) { } // Create a plan - s.plan = s.eval.MakePlan(s.job) + s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) // Reset the failed allocations s.failedTGAllocs = nil @@ -210,18 +215,18 @@ func (s *SystemScheduler) computeJobAllocs() error { // Add all the allocs to stop for _, e := range diff.stop { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocNotNeeded, "") + s.plan.AppendStoppedAlloc(e.Alloc, allocNotNeeded, "") } // Add all the allocs to migrate for _, e := range diff.migrate { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocNodeTainted, "") + s.plan.AppendStoppedAlloc(e.Alloc, allocNodeTainted, "") } // Lost allocations should be transitioned to desired status stop and client // status lost. for _, e := range diff.lost { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocLost, structs.AllocClientStatusLost) + s.plan.AppendStoppedAlloc(e.Alloc, allocLost, structs.AllocClientStatusLost) } // Attempt to do the upgrades in place @@ -351,7 +356,7 @@ func (s *SystemScheduler) computePlacements(place []allocTuple) error { if option.PreemptedAllocs != nil { var preemptedAllocIDs []string for _, stop := range option.PreemptedAllocs { - s.plan.AppendPreemptedAlloc(stop, structs.AllocDesiredStatusEvict, alloc.ID) + s.plan.AppendPreemptedAlloc(stop, alloc.ID) preemptedAllocIDs = append(preemptedAllocIDs, stop.ID) if s.eval.AnnotatePlan && s.plan.Annotations != nil { diff --git a/scheduler/testing.go b/scheduler/testing.go index f0501010246..61fa7f79c61 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -216,7 +216,7 @@ func (h *Harness) Snapshot() State { // a snapshot of current state using the harness for planning. func (h *Harness) Scheduler(factory Factory) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h) + return factory(logger, h.Snapshot(), h, false) } // Process is used to process an evaluation given a factory diff --git a/scheduler/util.go b/scheduler/util.go index c9700538167..becae99601b 100644 --- a/scheduler/util.go +++ b/scheduler/util.go @@ -507,8 +507,7 @@ func inplaceUpdate(ctx Context, eval *structs.Evaluation, job *structs.Job, // the current allocation is discounted when checking for feasibility. // Otherwise we would be trying to fit the tasks current resources and // updated resources. After select is called we can remove the evict. - ctx.Plan().AppendUpdate(update.Alloc, structs.AllocDesiredStatusStop, - allocInPlace, "") + ctx.Plan().AppendStoppedAlloc(update.Alloc, allocInPlace, "") // Attempt to match the task group option := stack.Select(update.TaskGroup, nil) // This select only looks at one node so we don't pass selectOptions @@ -573,7 +572,7 @@ func evictAndPlace(ctx Context, diff *diffResult, allocs []allocTuple, desc stri n := len(allocs) for i := 0; i < n && i < *limit; i++ { a := allocs[i] - ctx.Plan().AppendUpdate(a.Alloc, structs.AllocDesiredStatusStop, desc, "") + ctx.Plan().AppendStoppedAlloc(a.Alloc, desc, "") diff.place = append(diff.place, a) } if n <= *limit { @@ -734,7 +733,7 @@ func updateNonTerminalAllocsToLost(plan *structs.Plan, tainted map[string]*struc if alloc.DesiredStatus == structs.AllocDesiredStatusStop && (alloc.ClientStatus == structs.AllocClientStatusRunning || alloc.ClientStatus == structs.AllocClientStatusPending) { - plan.AppendUpdate(alloc, structs.AllocDesiredStatusStop, allocLost, structs.AllocClientStatusLost) + plan.AppendStoppedAlloc(alloc, allocLost, structs.AllocClientStatusLost) } } } @@ -784,7 +783,7 @@ func genericAllocUpdateFn(ctx Context, stack Stack, evalID string) allocUpdateTy // the current allocation is discounted when checking for feasibility. // Otherwise we would be trying to fit the tasks current resources and // updated resources. After select is called we can remove the evict. - ctx.Plan().AppendUpdate(existing, structs.AllocDesiredStatusStop, allocInPlace, "") + ctx.Plan().AppendStoppedAlloc(existing, allocInPlace, "") // Attempt to match the task group option := stack.Select(newTG, nil) // This select only looks at one node so we don't pass selectOptions From 50dc0ec2d4b5ebc2174f457257cea6ed6afbfb4a Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Tue, 5 Mar 2019 13:41:41 -0800 Subject: [PATCH 5/7] Add tests for plan normalization --- nomad/fsm_test.go | 2 +- nomad/plan_apply_test.go | 150 + nomad/state/state_store_test.go | 101 +- nomad/structs/structs_test.go | 147 +- nomad/util_test.go | 90 +- nomad/worker_test.go | 68 +- scheduler/generic_sched_test.go | 7850 ++++++++++++++++--------------- scheduler/system_sched_test.go | 3322 ++++++------- scheduler/testing.go | 74 +- scheduler/util_test.go | 12 +- 10 files changed, 6310 insertions(+), 5506 deletions(-) diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 0c3a846a85f..06177a0831d 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -56,7 +56,7 @@ func testFSM(t *testing.T) *nomadFSM { Logger: logger, Region: "global", } - fsm, err := NewFSM(fsmConfig) + fsm, err := NewFSM(fsmConfig, TestServer(t, nil)) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 4dfa7a43b02..619b7b9071d 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -3,6 +3,7 @@ package nomad import ( "reflect" "testing" + "time" memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" @@ -62,6 +63,7 @@ func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { } } +// Deprecated: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { t.Parallel() s1 := TestServer(t, nil) @@ -228,6 +230,154 @@ func TestPlanApply_applyPlan(t *testing.T) { assert.Equal(index, evalOut.ModifyIndex) } +func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.Build = "0.9.1" + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + // Register a fake deployment + oldDeployment := mock.Deployment() + if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { + t.Fatalf("UpsertDeployment failed: %v", err) + } + + // Create a deployment + dnew := mock.Deployment() + + // Create a deployment update for the old deployment id + desiredStatus, desiredStatusDescription := "foo", "bar" + updates := []*structs.DeploymentStatusUpdate{ + { + DeploymentID: oldDeployment.ID, + Status: desiredStatus, + StatusDescription: desiredStatusDescription, + }, + } + + // Register allocs, deployment and deployment update + alloc := mock.Alloc() + stoppedAlloc := mock.Alloc() + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "Desired Description", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) + s1.State().UpsertAllocs(1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + // Create an eval + eval := mock.Eval() + eval.JobID = alloc.JobID + if err := s1.State().UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + timestampBeforeCommit := time.Now().UTC().UnixNano() + planRes := &structs.PlanResult{ + NodeAllocation: map[string][]*structs.Allocation{ + node.ID: {alloc}, + }, + NodeUpdate: map[string][]*structs.Allocation{ + stoppedAlloc.NodeID: {stoppedAllocDiff}, + }, + NodePreemptions: map[string][]*structs.Allocation{ + preemptedAlloc.NodeID: {preemptedAllocDiff}, + }, + Deployment: dnew, + DeploymentUpdates: updates, + } + + // Snapshot the state + snap, err := s1.State().Snapshot() + if err != nil { + t.Fatalf("err: %v", err) + } + + // Create the plan with a deployment + plan := &structs.Plan{ + Job: alloc.Job, + Deployment: dnew, + DeploymentUpdates: updates, + EvalID: eval.ID, + } + + // Apply the plan + future, err := s1.applyPlan(plan, planRes, snap) + assert := assert.New(t) + assert.Nil(err) + + // Verify our optimistic snapshot is updated + ws := memdb.NewWatchSet() + allocOut, err := snap.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + + deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(deploymentOut) + + // Check plan does apply cleanly + index, err := planWaitFuture(future) + assert.Nil(err) + assert.NotEqual(0, index) + + // Lookup the allocation + fsmState := s1.fsm.State() + allocOut, err = fsmState.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + assert.True(allocOut.CreateTime > 0) + assert.True(allocOut.ModifyTime > 0) + assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) + + // Verify stopped alloc diff applied cleanly + updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedStoppedAlloc) + assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) + assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) + assert.Equal(updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) + + // Verify preempted alloc diff applied cleanly + updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedPreemptedAlloc) + assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedPreemptedAlloc.DesiredDescription, + "Preempted by alloc ID " + preemptedAllocDiff.PreemptedByAllocation) + assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) + + // Lookup the new deployment + dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(dout) + + // Lookup the updated deployment + dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) + assert.Nil(err) + assert.NotNil(dout2) + assert.Equal(desiredStatus, dout2.Status) + assert.Equal(desiredStatusDescription, dout2.StatusDescription) + + // Lookup updated eval + evalOut, err := fsmState.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.Equal(index, evalOut.ModifyIndex) +} + func TestPlanApply_EvalPlan_Simple(t *testing.T) { t.Parallel() state := testStateStore(t) diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 3cf3b977606..af248a55729 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -88,6 +88,7 @@ func TestStateStore_Blocking_MinQuery(t *testing.T) { } } +// COMPAT 0.11: Uses AllocUpdateRequest.Alloc // This test checks that: // 1) The job is denormalized // 2) Allocations are created @@ -140,6 +141,94 @@ func TestStateStore_UpsertPlanResults_AllocationsCreated_Denormalized(t *testing assert.EqualValues(1000, evalOut.ModifyIndex) } +// This test checks that: +// 1) The job is denormalized +// 2) Allocations are denormalized and updated with the diff +func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { + state := testStateStore(t) + alloc := mock.Alloc() + job := alloc.Job + alloc.Job = nil + + stoppedAlloc := mock.Alloc() + stoppedAlloc.Job = job + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "desired desc", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAlloc.Job = job + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + + if err := state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc}); err != nil { + t.Fatalf("err: %v", err) + } + + if err := state.UpsertJob(999, job); err != nil { + t.Fatalf("err: %v", err) + } + + eval := mock.Eval() + eval.JobID = job.ID + + // Create an eval + if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + // Create a plan result + res := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + AllocsUpdated: []*structs.Allocation{alloc}, + AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, + Job: job, + }, + EvalID: eval.ID, + NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, + } + assert := assert.New(t) + planModifyIndex := uint64(1000) + err := state.UpsertPlanResults(planModifyIndex, &res) + assert.Nil(err) + + ws := memdb.NewWatchSet() + out, err := state.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.Equal(alloc, out) + + updatedStoppedAlloc, err := state.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.Equal(stoppedAllocDiff.DesiredDescription, updatedStoppedAlloc.DesiredDescription) + assert.Equal(structs.AllocDesiredStatusStop, updatedStoppedAlloc.DesiredStatus) + assert.Equal(stoppedAllocDiff.ClientStatus, updatedStoppedAlloc.ClientStatus) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + + updatedPreemptedAlloc, err := state.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.Equal(structs.AllocDesiredStatusEvict, updatedPreemptedAlloc.DesiredStatus) + assert.Equal(preemptedAllocDiff.PreemptedByAllocation, updatedPreemptedAlloc.PreemptedByAllocation) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + + index, err := state.Index("allocs") + assert.Nil(err) + assert.EqualValues(planModifyIndex, index) + + if watchFired(ws) { + t.Fatalf("bad") + } + + evalOut, err := state.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.EqualValues(planModifyIndex, evalOut.ModifyIndex) +} + // This test checks that the deployment is created and allocations count towards // the deployment func TestStateStore_UpsertPlanResults_Deployment(t *testing.T) { @@ -271,11 +360,9 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { require.NoError(err) minimalPreemptedAlloc := &structs.Allocation{ - ID: preemptedAlloc.ID, - Namespace: preemptedAlloc.Namespace, - DesiredStatus: structs.AllocDesiredStatusEvict, - ModifyTime: time.Now().Unix(), - DesiredDescription: fmt.Sprintf("Preempted by allocation %v", alloc.ID), + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + ModifyTime: time.Now().Unix(), } // Create eval for preempted job @@ -316,7 +403,7 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { preempted, err := state.AllocByID(ws, preemptedAlloc.ID) require.NoError(err) require.Equal(preempted.DesiredStatus, structs.AllocDesiredStatusEvict) - require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by allocation %v", alloc.ID)) + require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by alloc ID %v", alloc.ID)) // Verify eval for preempted job preemptedJobEval, err := state.EvalByID(ws, eval2.ID) diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 90b33b19827..2cd3922aebf 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/hashicorp/consul/api" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" "github.com/kr/pretty" "github.com/stretchr/testify/assert" @@ -2842,6 +2842,151 @@ func TestTaskArtifact_Validate_Checksum(t *testing.T) { } } +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = true + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := &Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDesc, + ClientStatus: AllocClientStatusLost, + } + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsFalse(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = false + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := new(Allocation) + *expectedStoppedAlloc = *stoppedAlloc + expectedStoppedAlloc.DesiredDescription = desiredDesc + expectedStoppedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedStoppedAlloc.ClientStatus = AllocClientStatusLost + expectedStoppedAlloc.Job = nil + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: preemptedAlloc.JobID, + Namespace: preemptedAlloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: preemptedAlloc.AllocatedResources, + TaskResources: preemptedAlloc.TaskResources, + SharedResources: preemptedAlloc.SharedResources, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_AppendStoppedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + } + alloc := MockAlloc() + desiredDesc := "Desired desc" + + plan.AppendStoppedAlloc(alloc, desiredDesc, AllocClientStatusLost) + + appendedAlloc := plan.NodeUpdate[alloc.NodeID][0] + expectedAlloc := new(Allocation) + *expectedAlloc = *alloc + expectedAlloc.DesiredDescription = desiredDesc + expectedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedAlloc.ClientStatus = AllocClientStatusLost + expectedAlloc.Job = nil + assert.Equal(t, expectedAlloc, appendedAlloc) + assert.Equal(t, alloc.Job, plan.Job) +} + +func TestPlan_AppendPreemptedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodePreemptions: make(map[string][]*Allocation), + } + alloc := MockAlloc() + preemptingAllocID := uuid.Generate() + + plan.AppendPreemptedAlloc(alloc, preemptingAllocID) + + appendedAlloc := plan.NodePreemptions[alloc.NodeID][0] + expectedAlloc := &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: alloc.JobID, + Namespace: alloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: alloc.AllocatedResources, + TaskResources: alloc.TaskResources, + SharedResources: alloc.SharedResources, + } + assert.Equal(t, expectedAlloc, appendedAlloc) +} + +func TestPlan_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Plan{}) + + msgPackTags, _ := planType.FieldByName("_struct") + normalizeTag, _ := planType.FieldByName("NormalizeAllocs") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) + assert.Equal(t, normalizeTag.Tag, reflect.StructTag(`codec:"-"`)) +} + +func TestAllocation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Allocation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + +func TestEvaluation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Evaluation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + func TestAllocation_Terminated(t *testing.T) { type desiredState struct { ClientStatus string diff --git a/nomad/util_test.go b/nomad/util_test.go index 5614067609c..f216d5e4243 100644 --- a/nomad/util_test.go +++ b/nomad/util_test.go @@ -86,23 +86,8 @@ func TestIsNomadServer(t *testing.T) { } } -func TestServersMeetMinimumVersion(t *testing.T) { +func TestServersMeetMinimumVersionExcludingFailed(t *testing.T) { t.Parallel() - makeMember := func(version string) serf.Member { - return serf.Member{ - Name: "foo", - Addr: net.IP([]byte{127, 0, 0, 1}), - Tags: map[string]string{ - "role": "nomad", - "region": "aws", - "dc": "east-aws", - "port": "10000", - "build": version, - "vsn": "1", - }, - Status: serf.StatusAlive, - } - } cases := []struct { members []serf.Member @@ -112,7 +97,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, meets reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -120,7 +105,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server in dev, meets reqs { members: []serf.Member{ - makeMember("0.8.5-dev"), + makeMember("0.8.5-dev", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -128,7 +113,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server with meta, meets reqs { members: []serf.Member{ - makeMember("0.7.5+ent"), + makeMember("0.7.5+ent", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -136,16 +121,17 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, doesn't meet reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, }, - // Multiple servers, meets req version + // Multiple servers, meets req version, includes failed that doesn't meet req { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -153,8 +139,8 @@ func TestServersMeetMinimumVersion(t *testing.T) { // Multiple servers, doesn't meet req version { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, @@ -169,6 +155,60 @@ func TestServersMeetMinimumVersion(t *testing.T) { } } +func TestServersMeetMinimumVersionIncludingFailed(t *testing.T) { + t.Parallel() + + cases := []struct { + members []serf.Member + ver *version.Version + expected bool + }{ + // Multiple servers, meets req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.5", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: true, + }, + // Multiple servers, doesn't meet req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: false, + }, + } + + for _, tc := range cases { + result := ServersMeetMinimumVersion(tc.members, tc.ver, true) + if result != tc.expected { + t.Fatalf("bad: %v, %v, %v", result, tc.ver.String(), tc) + } + } +} + +func makeMember(version string, status serf.MemberStatus) serf.Member { + return serf.Member{ + Name: "foo", + Addr: net.IP([]byte{127, 0, 0, 1}), + Tags: map[string]string{ + "role": "nomad", + "region": "aws", + "dc": "east-aws", + "port": "10000", + "build": version, + "vsn": "1", + }, + Status: status, + } +} + func TestShuffleStrings(t *testing.T) { t.Parallel() // Generate input diff --git a/nomad/worker_test.go b/nomad/worker_test.go index b7a9e526a21..b2dc94f22b8 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -8,20 +8,22 @@ import ( "time" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/scheduler" "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/assert" ) type NoopScheduler struct { - state scheduler.State - planner scheduler.Planner - eval *structs.Evaluation - err error + state scheduler.State + planner scheduler.Planner + eval *structs.Evaluation + allowPlanOptimization bool + err error } func (n *NoopScheduler) Process(eval *structs.Evaluation) error { @@ -38,8 +40,9 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { func init() { scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { n := &NoopScheduler{ - state: s, - planner: p, + state: s, + planner: p, + allowPlanOptimization: allowPlanOptimization, } return n } @@ -390,6 +393,57 @@ func TestWorker_SubmitPlan(t *testing.T) { } } +func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 + c.EnabledSchedulers = []string{structs.JobTypeService} + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + job := mock.Job() + eval1 := mock.Eval() + eval1.JobID = job.ID + s1.fsm.State().UpsertJob(0, job) + s1.fsm.State().UpsertEvals(0, []*structs.Evaluation{eval1}) + + stoppedAlloc := mock.Alloc() + preemptedAlloc := mock.Alloc() + s1.fsm.State().UpsertAllocs(5, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + + // Create an allocation plan + plan := &structs.Plan{ + Job: job, + EvalID: eval1.ID, + NodeUpdate: make(map[string][]*structs.Allocation), + NodePreemptions: make(map[string][]*structs.Allocation), + NormalizeAllocs: true, + } + desiredDescription := "desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost) + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + // Attempt to submit a plan + w := &Worker{srv: s1, logger: s1.logger} + w.SubmitPlan(plan) + + assert.Equal(t, &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + }, plan.NodePreemptions[preemptedAlloc.NodeID][0]) + assert.Equal(t, &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDescription, + ClientStatus: structs.AllocClientStatusLost, + }, plan.NodeUpdate[stoppedAlloc.NodeID][0]) +} + func TestWorker_SubmitPlan_MissingNodeRefresh(t *testing.T) { t.Parallel() s1 := TestServer(t, func(c *Config) { diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 3613b4df8b3..223d41d5bfe 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -1,661 +1,336 @@ package scheduler import ( - "fmt" - "reflect" + "fmt" + "reflect" "sort" - "testing" - "time" - - memdb "github.com/hashicorp/go-memdb" - "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/mock" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "testing" + "time" + + memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestServiceSched_JobRegister(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval has no spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different ports were used. - used := make(map[int]map[string]struct{}) - for _, alloc := range out { - for _, resource := range alloc.TaskResources { - for _, port := range resource.Networks[0].DynamicPorts { - nodeMap, ok := used[port.Value] - if !ok { - nodeMap = make(map[string]struct{}) - used[port.Value] = nodeMap - } - if _, ok := nodeMap[alloc.NodeID]; ok { - t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) - } - nodeMap[alloc.NodeID] = struct{}{} - } - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) +func IsPlanOptimizedStr(allowPlanOptimization bool) string { + return fmt.Sprintf("Is plan optimized: %v", allowPlanOptimization) } -func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } +func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure the plan allocated - plan := h.Plans[0] - planned := make(map[string]*structs.Allocation) - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - planned[alloc.ID] = alloc - } - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Update the job to force a rolling upgrade - updated := job.Copy() - updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 - noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a job that uses distinct host and has count 1 higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure we have created only one new allocation - // Ensure a single plan - if len(h1.Plans) != 1 { - t.Fatalf("bad: %#v", h1.Plans) - } - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 10 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocations were placed on the same node as the older - // ones - for _, new := range newPlanned { - if new.PreviousAllocation == "" { - t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) - } - - old, ok := planned[new.PreviousAllocation] - if !ok { - t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) - } - if new.NodeID != old.NodeID { - t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) - } - } -} + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } -func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job with count 2 and disk as 60GB so that only one allocation - // can fit - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Ensure the eval has a blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the plan allocated only one allocation - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } - // Ensure only one allocation was placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { - h := NewHarness(t) +func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + rack := "rack2" + if i < 5 { + rack = "rack1" + } + node.Meta["rack"] = rack + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct host and has count 1 higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 8 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 4 { + t.Fatalf("bad: %#v", plan) + } - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure all allocations placed + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } -func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - rack := "rack2" - if i < 5 { - rack = "rack1" - } - node.Meta["rack"] = rack - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure each node was only used twice + used := make(map[string]uint64) + for _, alloc := range out { + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) + } + used[alloc.NodeID]++ + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 8 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - RTarget: "2", + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 4 { - t.Fatalf("bad: %#v", plan) } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } - - // Ensure each node was only used twice - used := make(map[string]uint64) - for _, alloc := range out { - if count, _ := used[alloc.NodeID]; count > 2 { - t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) - } - used[alloc.NodeID]++ - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 2; i++ { - node := mock.Node() - node.Meta["ssd"] = "true" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 2; i++ { + node := mock.Node() + node.Meta["ssd"] = "true" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property only on one task group. - job := mock.Job() - job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.ssd}", - }) + // Create a job that uses distinct property only on one task group. + job := mock.Job() + job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.ssd}", + }) - job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + job.TaskGroups[1].Name = "tg2" + job.TaskGroups[1].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure all allocations placed + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { - h := NewHarness(t) - assert := assert.New(t) - - // Create a job that uses distinct property over the node-id - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${node.unique.id}", + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 6; i++ { - node := mock.Node() - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - } - - // Create some allocations - var allocs []*structs.Allocation - for i := 0; i < 3; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") - - // Update the count - job2 := job.Copy() - job2.TaskGroups[0].Count = 6 - assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) } - assert.Len(planned, 6, "Planned Allocations") - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - assert.Nil(err, "AllocsByJob") - - // Ensure all allocations placed - assert.Len(out, 6, "Placed Allocations") - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// Test job registration with spread configured -func TestServiceSched_Spread(t *testing.T) { - assert := assert.New(t) - - start := uint8(100) - step := uint8(10) +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { + for _, allowPlanOptimization := range []bool{false, true} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + assert := assert.New(t) - for i := 0; i < 10; i++ { - name := fmt.Sprintf("%d%% in dc1", start) - t.Run(name, func(t *testing.T) { - h := NewHarness(t) - remaining := uint8(100 - start) - // Create a job that uses spread over data center + // Create a job that uses distinct property over the node-id job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - SpreadTarget: []*structs.SpreadTarget{ - { - Value: "dc1", - Percent: start, - }, - { - Value: "dc2", - Percent: remaining, - }, - }, + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${node.unique.id}", }) assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 + + // Create some nodes var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { + for i := 0; i < 6; i++ { node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } nodes = append(nodes, node) assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node } + // Create some allocations + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") + + // Update the count + job2 := job.Copy() + job2.TaskGroups[0].Count = 6 + assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") + // Create a mock evaluation to register the job eval := &structs.Evaluation{ Namespace: structs.DefaultNamespace, @@ -682,3697 +357,4237 @@ func TestServiceSched_Spread(t *testing.T) { // Ensure the plan allocated var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { + for _, allocList := range plan.NodeAllocation { planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c } - assert.Len(planned, 10, "Planned Allocations") + assert.Len(planned, 6, "Planned Allocations") - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 10 - i - if i > 0 { - expectedCounts["dc2"] = i - } - require.Equal(t, expectedCounts, dcAllocsMap) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + assert.Nil(err, "AllocsByJob") + + // Ensure all allocations placed + assert.Len(out, 6, "Placed Allocations") h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - start = start - step } } -// Test job registration with even spread across dc -func TestServiceSched_EvenSpread(t *testing.T) { - assert := assert.New(t) - - h := NewHarness(t) - // Create a job that uses even spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c - } - assert.Len(planned, 10, "Planned Allocations") - - // Expect even split allocs across datacenter - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 5 - expectedCounts["dc2"] = 5 +// Test job registration with spread configured +func TestServiceSched_Spread(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) - require.Equal(t, expectedCounts, dcAllocsMap) + start := uint8(100) + step := uint8(10) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%d%% in dc1", start) + t.Run(name, func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + remaining := uint8(100 - start) + // Create a job that uses spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + SpreadTarget: []*structs.SpreadTarget{ + { + Value: "dc1", + Percent: start, + }, + { + Value: "dc2", + Percent: remaining, + }, + }, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") + + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 10 - i + if i > 0 { + expectedCounts["dc2"] = i + } + require.Equal(t, expectedCounts, dcAllocsMap) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + start = start - step + } + }) + } } -func TestServiceSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) +// Test job registration with even spread across dc +func TestServiceSched_EvenSpread(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t, allowPlanOptimization) + // Create a job that uses even spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Expect even split allocs across datacenter + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 5 + expectedCounts["dc2"] = 5 - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + require.Equal(t, expectedCounts, dcAllocsMap) - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } +func TestServiceSched_JobRegister_Annotate(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - expected := &structs.DesiredUpdates{Place: 10} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) - } -} + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } -func TestServiceSched_JobRegister_CountZero(t *testing.T) { - h := NewHarness(t) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + h.AssertEvalStatus(t, structs.EvalStatusComplete) -func TestServiceSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) - - // Create NO nodes - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) + expected := &structs.DesiredUpdates{Place: 10} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + } + }) } +} - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } +func TestServiceSched_JobRegister_CountZero(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { - t.Fatalf("bad: %#v", metrics) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Check queued allocations - queued := outEval.QueuedAllocations["web"] - if queued != 10 { - t.Fatalf("expected queued: %v, actual: %v", 10, queued) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { - h := NewHarness(t) - - // Create a full node - node := mock.Node() - node.ReservedResources = &structs.NodeReservedResources{ - Cpu: structs.NodeReservedCpuResources{ - CpuShares: node.NodeResources.Cpu.CpuShares, - }, - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create an ineligible node - node2 := mock.Node() - node2.Attributes["kernel.name"] = "windows" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a jobs - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } +func TestServiceSched_JobRegister_AllocFail(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create NO nodes + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan has created a follow up eval. - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - created := h.CreateEvals[0] - if created.Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", created) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - classes := created.ClassEligibility - if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { - t.Fatalf("bad: %#v", classes) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - if created.EscapedComputedClass { - t.Fatalf("bad: %#v", created) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { + t.Fatalf("bad: %#v", metrics) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { - t.Fatalf("bad: %#v", metrics) + // Check queued allocations + queued := outEval.QueuedAllocations["web"] + if queued != 10 { + t.Fatalf("expected queued: %v, actual: %v", 10, queued) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { - h := NewHarness(t) - - // Create one node - node := mock.Node() - node.NodeClass = "class_0" - noErr(t, node.ComputeClass()) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job that constrains on a node class - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].Constraints = append(job.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "class_0", - Operand: "=", - }, - ) - tg2 := job.TaskGroups[0].Copy() - tg2.Name = "web2" - tg2.Constraints[1].RTarget = "class_1" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a full node + node := mock.Node() + node.ReservedResources = &structs.NodeReservedResources{ + Cpu: structs.NodeReservedCpuResources{ + CpuShares: node.NodeResources.Cpu.CpuShares, + }, + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Create an ineligible node + node2 := mock.Node() + node2.Attributes["kernel.name"] = "windows" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 2 { - t.Fatalf("bad: %#v", plan) - } + // Create a jobs + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure two allocations placed - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan failed to alloc one tg - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - metrics, ok := outEval.FailedTGAllocs[tg2.Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Ensure the plan has created a follow up eval. + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Check the coalesced failures - if metrics.CoalescedFailures != tg2.Count-1 { - t.Fatalf("bad: %#v", metrics) - } + created := h.CreateEvals[0] + if created.Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", created) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + classes := created.ClassEligibility + if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { + t.Fatalf("bad: %#v", classes) + } -// This test just ensures the scheduler handles the eval type to avoid -// regressions. -func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { - h := NewHarness(t) - - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerMaxPlans, - JobID: job.ID, - } + if created.EscapedComputedClass { + t.Fatalf("bad: %#v", created) + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } -func TestServiceSched_Plan_Partial_Progress(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job with a high resource ask so that all the allocations can't - // be placed on a single node. - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { + t.Fatalf("bad: %#v", metrics) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] +func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Create one node + node := mock.Node() + node.NodeClass = "class_0" + noErr(t, node.ComputeClass()) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Create a job that constrains on a node class + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].Constraints = append(job.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "class_0", + Operand: "=", + }, + ) + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "web2" + tg2.Constraints[1].RTarget = "class_1" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure only one allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 2 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure two allocations placed + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } -func TestServiceSched_EvaluateBlockedEval(t *testing.T) { - h := NewHarness(t) - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan failed to alloc one tg + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + metrics, ok := outEval.FailedTGAllocs[tg2.Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Ensure that the eval was reblocked - if len(h.ReblockEvals) != 1 { - t.Fatalf("bad: %#v", h.ReblockEvals) - } - if h.ReblockEvals[0].ID != eval.ID { - t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) - } + // Check the coalesced failures + if metrics.CoalescedFailures != tg2.Count-1 { + t.Fatalf("bad: %#v", metrics) + } - // Ensure the eval status was not updated - if len(h.Evals) != 0 { - t.Fatalf("Existing eval should not have status set") + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } } -func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job and set the task group count to zero. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } +// This test just ensures the scheduler handles the eval type to avoid +// regressions. +func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerMaxPlans, + JobID: job.ID, + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the eval has no spawned blocked eval - if len(h.Evals) != 1 { - t.Fatalf("bad: %#v", h.Evals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) +func TestServiceSched_Plan_Partial_Progress(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure the eval was not reblocked - if len(h.ReblockEvals) != 0 { - t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") - } + // Create a job with a high resource ask so that all the allocations can't + // be placed on a single node. + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure queued allocations is zero - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued: %v, actual: %v", 0, queued) - } -} + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) -func TestServiceSched_JobModify(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure only one allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// Have a single node and submit a job. Increment the count such that all fit -// on the node but the node doesn't have enough resources to fit the new count + -// 1. This tests that we properly discount the resources of existing allocs. -func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { - h := NewHarness(t) +func TestServiceSched_EvaluateBlockedEval(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create one node - node := mock.Node() - node.NodeResources.Cpu.CpuShares = 1000 - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Generate a fake job with one allocation - job := mock.Job() - job.TaskGroups[0].Tasks[0].Resources.CPU = 256 - job2 := job.Copy() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - var allocs []*structs.Allocation - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job to count 3 - job2.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan didn't evicted the alloc - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure that the eval was reblocked + if len(h.ReblockEvals) != 1 { + t.Fatalf("bad: %#v", h.ReblockEvals) + } + if h.ReblockEvals[0].ID != eval.ID { + t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) + // Ensure the eval status was not updated + if len(h.Evals) != 0 { + t.Fatalf("Existing eval should not have status set") + } + }) } +} - // Ensure the plan had no failures - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - if outEval == nil || len(outEval.FailedTGAllocs) != 0 { - t.Fatalf("bad: %#v", outEval) - } +func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + // Create a job and set the task group count to zero. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } -func TestServiceSched_JobModify_CountZero(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job to be count zero - job2 := mock.Job() - job2.ID = job.ID - job2.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the eval has no spawned blocked eval + if len(h.Evals) != 1 { + t.Fatalf("bad: %#v", h.Evals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan didn't allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure the eval was not reblocked + if len(h.ReblockEvals) != 0 { + t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") + } -func TestServiceSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, + // Ensure queued allocations is zero + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued: %v, actual: %v", 0, queued) + } + }) } +} - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestServiceSched_JobModify(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != desiredUpdates { - t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Update the job + job2 := mock.Job() + job2.ID = job.ID - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } -} + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) -// This tests that the old allocation is stopped before placing. -// It is critical to test that the updated job attempts to place more -// allocations as this allows us to assert that destructive changes are done -// first. -func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { - h := NewHarness(t) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a node and clear the reserved resources - node := mock.Node() - node.ReservedResources = nil - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a resource ask that is the same as the resources available on the - // node - cpu := node.NodeResources.Cpu.CpuShares - mem := node.NodeResources.Memory.MemoryMB + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - request := &structs.Resources{ - CPU: int(cpu), - MemoryMB: int(mem), - } - allocated := &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: cpu, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: mem, + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +// Have a single node and submit a job. Increment the count such that all fit +// on the node but the node doesn't have enough resources to fit the new count + +// 1. This tests that we properly discount the resources of existing allocs. +func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create one node + node := mock.Node() + node.NodeResources.Cpu.CpuShares = 1000 + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with one allocation + job := mock.Job() + job.TaskGroups[0].Tasks[0].Resources.CPU = 256 + job2 := job.Copy() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job to count 3 + job2.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan didn't evicted the alloc + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan had no failures + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + if outEval == nil || len(outEval.FailedTGAllocs) != 0 { + t.Fatalf("bad: %#v", outEval) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobModify_CountZero(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job to be count zero + job2 := mock.Job() + job2.ID = job.ID + job2.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan didn't allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobModify_Rolling(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != desiredUpdates { + t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) + } +} + +// This tests that the old allocation is stopped before placing. +// It is critical to test that the updated job attempts to place more +// allocations as this allows us to assert that destructive changes are done +// first. +func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a node and clear the reserved resources + node := mock.Node() + node.ReservedResources = nil + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a resource ask that is the same as the resources available on the + // node + cpu := node.NodeResources.Cpu.CpuShares + mem := node.NodeResources.Memory.MemoryMB + + request := &structs.Resources{ + CPU: int(cpu), + MemoryMB: int(mem), + } + allocated := &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: cpu, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: mem, + }, + }, }, - }, - }, + } + + // Generate a fake job with one alloc that consumes the whole node + job := mock.Job() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Resources = request + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.AllocatedResources = allocated + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Update the job to place more versions of the task group, drop the count + // and force destructive updates + job2 := job.Copy() + job2.TaskGroups[0].Count = 5 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 1 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 5 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) } +} + +func TestServiceSched_JobModify_Canaries(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 2 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + Canary: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted nothing + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { + t.Fatalf("bad: %#v", state) + } + + // Assert the canaries were added to the placed list + if len(state.PlacedCanaries) != desiredUpdates { + t.Fatalf("bad: %#v", state) + } + }) + } +} + +func TestServiceSched_JobModify_InPlace(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and create an older deployment + job := mock.Job() + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create allocs that are part of the old deployment + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } + } + + // Verify the deployment id was changed and health cleared + for _, alloc := range out { + if alloc.DeploymentID == d.ID { + t.Fatalf("bad: deployment id not cleared") + } else if alloc.DeploymentStatus != nil { + t.Fatalf("bad: deployment status not cleared") + } + } + }) + } +} + +func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + node.Meta["rack"] = fmt.Sprintf("rack%d", i) + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + oldJob := job.Copy() + oldJob.JobModifyIndex -= 1 + oldJob.TaskGroups[0].Count = 4 + + // Place 4 of 10 + var allocs []*structs.Allocation + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = oldJob + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", planned) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobDeregister_Purged(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Generate a fake job with allocations + job := mock.Job() + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all nodes + if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + if alloc.Job == nil { + t.Fatalf("bad: %#v", alloc) + } + } + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobDeregister_Stopped(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + + // Generate a fake job with allocations + job := mock.Job() + job.Stop = true + require.NoError(h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a summary where the queued allocs are set as we want to assert + // they get zeroed out. + summary := mock.JobSummary(job.ID) + web := summary.Summary["web"] + web.Queued = 2 + require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.NoError(h.Process(NewServiceScheduler, eval)) + + // Ensure a single plan + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan evicted all nodes + require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(err) + + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + require.NotNil(alloc.Job) + } + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + require.Empty(out) + + // Assert the job summary is cleared out + sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) + require.NoError(err) + require.NotNil(sout) + require.Contains(sout.Summary, "web") + webOut := sout.Summary["web"] + require.Zero(webOut.Queued) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDown(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + + // Cover each terminal case and ensure it doesn't change to lost + allocs[7].DesiredStatus = structs.AllocDesiredStatusRun + allocs[7].ClientStatus = structs.AllocClientStatusLost + allocs[8].DesiredStatus = structs.AllocDesiredStatusRun + allocs[8].ClientStatus = structs.AllocClientStatusFailed + allocs[9].DesiredStatus = structs.AllocDesiredStatusRun + allocs[9].ClientStatus = structs.AllocClientStatusComplete + + // Mark some allocs as running + for i := 0; i < 4; i++ { + out := allocs[i] + out.ClientStatus = structs.AllocClientStatusRunning + } + + // Mark appropriate allocs for migration + for i := 0; i < 7; i++ { + out := allocs[i] + out.DesiredTransition.Migrate = helper.BoolToPtr(true) + } + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Test the scheduler marked all non-terminal allocations as lost + if len(plan.NodeUpdate[node.ID]) != 7 { + t.Fatalf("bad: %#v", plan) + } + + for _, out := range plan.NodeUpdate[node.ID] { + if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad alloc: %#v", out) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeUpdate(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Mark some allocs as running + ws := memdb.NewWatchSet() + for i := 0; i < 4; i++ { + out, _ := h.State.AllocByID(ws, allocs[i].ID) + out.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) + } + + // Create a mock evaluation which won't trigger any new placements + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDrain(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDrain_Down(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Set the desired state of the allocs to stop + var stop []*structs.Allocation + for i := 0; i < 6; i++ { + newAlloc := allocs[i].Copy() + newAlloc.ClientStatus = structs.AllocDesiredStatusStop + newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + stop = append(stop, newAlloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) + + // Mark some of the allocations as running + var running []*structs.Allocation + for i := 4; i < 6; i++ { + newAlloc := stop[i].Copy() + newAlloc.ClientStatus = structs.AllocClientStatusRunning + running = append(running, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) + + // Mark some of the allocations as complete + var complete []*structs.Allocation + for i := 6; i < 10; i++ { + newAlloc := allocs[i].Copy() + newAlloc.TaskStates = make(map[string]*structs.TaskState) + newAlloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + newAlloc.ClientStatus = structs.AllocClientStatusComplete + complete = append(complete, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - // Generate a fake job with one alloc that consumes the whole node - job := mock.Job() - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Tasks[0].Resources = request - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - alloc := mock.Alloc() - alloc.AllocatedResources = allocated - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Update the job to place more versions of the task group, drop the count - // and force destructive updates - job2 := job.Copy() - job2.TaskGroups[0].Count = 5 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 1 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 6 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 5 { - t.Fatalf("bad: %#v", plan) - } + // Ensure that all the allocations which were in running or pending state + // has been marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + sort.Strings(lostAllocs) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + var expectedLostAllocs []string + for i := 0; i < 6; i++ { + expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) + } + sort.Strings(expectedLostAllocs) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { + t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) + } - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } } -func TestServiceSched_JobModify_Canaries(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 2 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - Canary: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Register a draining node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure the plan evicted nothing - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - for _, canary := range planned { - if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { - t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) - } - } + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { - t.Fatalf("bad: %#v", state) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Assert the canaries were added to the placed list - if len(state.PlacedCanaries) != desiredUpdates { - t.Fatalf("bad: %#v", state) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } + }) } } -func TestServiceSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and create an older deployment - job := mock.Job() - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create allocs that are part of the old deployment - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = d.ID - alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] +func TestServiceSched_RetryLimit(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) } - } - } - // Verify the deployment id was changed and health cleared - for _, alloc := range out { - if alloc.DeploymentID == d.ID { - t.Fatalf("bad: deployment id not cleared") - } else if alloc.DeploymentStatus != nil { - t.Fatalf("bad: deployment status not cleared") - } - } -} + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) -func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - node.Meta["rack"] = fmt.Sprintf("rack%d", i) - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - oldJob := job.Copy() - oldJob.JobModifyIndex -= 1 - oldJob.TaskGroups[0].Count = 4 - - // Place 4 of 10 - var allocs []*structs.Allocation - for i := 0; i < 4; i++ { - alloc := mock.Alloc() - alloc.Job = oldJob - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +} - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +func TestServiceSched_Reschedule_OnceNow(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: 5 * time.Second, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", planned) - } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) -func TestServiceSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) + // Mark this alloc as failed again, should not get rescheduled + newAlloc.ClientStatus = structs.AllocClientStatusFailed - // Generate a fake job with allocations - job := mock.Job() + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) + // Process the evaluation + err = h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created this time + out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(3, len(out)) + }) } +} - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] +// Tests that alloc reschedulable at a future time creates a follow up eval +func TestServiceSched_Reschedule_Later(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan evicted all nodes - if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + delayDuration := 15 * time.Second + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: delayDuration, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - if alloc.Job == nil { - t.Fatalf("bad: %#v", alloc) - } - } + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now}} + failedAllocID := allocs[1].ID - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) -func TestServiceSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - - // Generate a fake job with allocations - job := mock.Job() - job.Stop = true - require.NoError(h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a summary where the queued allocs are set as we want to assert - // they get zeroed out. - summary := mock.JobSummary(job.ID) - web := summary.Summary["web"] - web.Queued = 2 - require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - require.NoError(h.Process(NewServiceScheduler, eval)) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a single plan - require.Len(h.Plans, 1) - plan := h.Plans[0] + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the plan evicted all nodes - require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) + // Verify no new allocs were created + require.Equal(2, len(out)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(err) + // Verify follow up eval was created for the failed alloc + alloc, err := h.State.AllocByID(ws, failedAllocID) + require.Nil(err) + require.NotEmpty(alloc.FollowupEvalID) - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - require.NotNil(alloc.Job) + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { + t.Fatalf("bad: %#v", h.CreateEvals) + } + followupEval := h.CreateEvals[0] + require.Equal(now.Add(delayDuration), followupEval.WaitUntil) + }) } +} + +func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + maxRestartAttempts := 3 + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: maxRestartAttempts, + Interval: 30 * time.Minute, + Delay: 5 * time.Second, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - require.Empty(out) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Assert the job summary is cleared out - sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) - require.NoError(err) - require.NotNil(sout) - require.Contains(sout.Summary, "web") - webOut := sout.Summary["web"] - require.Zero(webOut.Queued) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) -func TestServiceSched_NodeDown(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Cover each terminal case and ensure it doesn't change to lost - allocs[7].DesiredStatus = structs.AllocDesiredStatusRun - allocs[7].ClientStatus = structs.AllocClientStatusLost - allocs[8].DesiredStatus = structs.AllocDesiredStatusRun - allocs[8].ClientStatus = structs.AllocClientStatusFailed - allocs[9].DesiredStatus = structs.AllocDesiredStatusRun - allocs[9].ClientStatus = structs.AllocClientStatusComplete - - // Mark some allocs as running - for i := 0; i < 4; i++ { - out := allocs[i] - out.ClientStatus = structs.AllocClientStatusRunning - } + expectedNumAllocs := 3 + expectedNumReschedTrackers := 1 - // Mark appropriate allocs for migration - for i := 0; i < 7; i++ { - out := allocs[i] - out.DesiredTransition.Migrate = helper.BoolToPtr(true) - } + failedAllocId := allocs[1].ID + failedNodeID := allocs[1].NodeID - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + assert := assert.New(t) + for i := 0; i < maxRestartAttempts; i++ { + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + noErr(t, err) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that a new allocation got created with its restart tracker info + assert.Equal(expectedNumAllocs, len(out)) + + // Find the new alloc with ClientStatusPending + var pendingAllocs []*structs.Allocation + var prevFailedAlloc *structs.Allocation + + for _, alloc := range out { + if alloc.ClientStatus == structs.AllocClientStatusPending { + pendingAllocs = append(pendingAllocs, alloc) + } + if alloc.ID == failedAllocId { + prevFailedAlloc = alloc + } + } + assert.Equal(1, len(pendingAllocs)) + newAlloc := pendingAllocs[0] + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) + + // Verify the previous NodeID in the most recent reschedule event + reschedEvents := newAlloc.RescheduleTracker.Events + assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) + assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) + + // Verify that the next alloc of the failed alloc is the newly rescheduled alloc + assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) + + // Mark this alloc as failed again + newAlloc.ClientStatus = structs.AllocClientStatusFailed + newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-12 * time.Second), + FinishedAt: now.Add(-10 * time.Second)}} + + failedAllocId = newAlloc.ID + failedNodeID = newAlloc.NodeID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + expectedNumAllocs += 1 + expectedNumReschedTrackers += 1 + } - // Test the scheduler marked all non-terminal allocations as lost - if len(plan.NodeUpdate[node.ID]) != 7 { - t.Fatalf("bad: %#v", plan) - } + // Process last eval again, should not reschedule + err := h.Process(NewServiceScheduler, eval) + assert.Nil(err) - for _, out := range plan.NodeUpdate[node.ID] { - if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad alloc: %#v", out) - } + // Verify no new allocs were created because restart attempts were exhausted + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Mark some allocs as running - ws := memdb.NewWatchSet() - for i := 0; i < 4; i++ { - out, _ := h.State.AllocByID(ws, allocs[i].ID) - out.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) - } - - // Create a mock evaluation which won't trigger any new placements - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) - } +// Tests that old reschedule attempts are pruned +func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } -func TestServiceSched_NodeDrain(t *testing.T) { - h := NewHarness(t) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + DelayFunction: "exponential", + MaxDelay: 1 * time.Hour, + Delay: 5 * time.Second, + Unlimited: true, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + now := time.Now() + // Mark allocations as failed with restart info + allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-15 * time.Minute)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + allocs[1].RescheduleTracker = &structs.RescheduleTracker{ + Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + Delay: 5 * time.Second, + }, + {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 10 * time.Second, + }, + {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 20 * time.Second, + }, + {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 40 * time.Second, + }, + {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 80 * time.Second, + }, + {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 160 * time.Second, + }, + }, + } + expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] + expectedDelay := 320 * time.Second + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + // Verify that the new alloc copied the last 5 reschedule attempts + assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) + mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] + // Verify that the failed alloc ID is in the most recent reschedule event + assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) + // Verify that the delay value was captured correctly + assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Set the desired state of the allocs to stop - var stop []*structs.Allocation - for i := 0; i < 6; i++ { - newAlloc := allocs[i].Copy() - newAlloc.ClientStatus = structs.AllocDesiredStatusStop - newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - stop = append(stop, newAlloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) - - // Mark some of the allocations as running - var running []*structs.Allocation - for i := 4; i < 6; i++ { - newAlloc := stop[i].Copy() - newAlloc.ClientStatus = structs.AllocClientStatusRunning - running = append(running, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) - - // Mark some of the allocations as complete - var complete []*structs.Allocation - for i := 6; i < 10; i++ { - newAlloc := allocs[i].Copy() - newAlloc.TaskStates = make(map[string]*structs.TaskState) - newAlloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - newAlloc.ClientStatus = structs.AllocClientStatusComplete - complete = append(complete, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and a reschedule policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + } + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) + + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed + } + + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = deployment.ID + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + }) + } + }) } +} - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] +func TestBatchSched_Run_CompleteAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 6 { - t.Fatalf("bad: %#v", plan) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure that all the allocations which were in running or pending state - // has been marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - sort.Strings(lostAllocs) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - var expectedLostAllocs []string - for i := 0; i < 6; i++ { - expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) - } - sort.Strings(expectedLostAllocs) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { - t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure no plan as it should be a no-op + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } -func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure no allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } } -func TestServiceSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} +func TestBatchSched_Run_FailedAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) -} + // Ensure a replacement alloc was placed. + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } -func TestServiceSched_Reschedule_OnceNow(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: 5 * time.Second, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +} - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +func TestBatchSched_Run_LostAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) - - // Mark this alloc as failed again, should not get rescheduled - newAlloc.ClientStatus = structs.AllocClientStatusFailed - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job + job := mock.Job() + job.ID = "my-job" + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err = h.Process(NewServiceScheduler, eval) - assert.Nil(err) - // Verify no new allocs were created this time - out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(3, len(out)) + // Desired = 3 + // Mark one as lost and then schedule + // [(0, run, running), (1, run, running), (1, stop, lost)] -} + // Create two running allocations + var allocs []*structs.Allocation + for i := 0; i <= 1; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } -// Tests that alloc reschedulable at a future time creates a follow up eval -func TestServiceSched_Reschedule_Later(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[1]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - delayDuration := 15 * time.Second - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: delayDuration, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now}} - failedAllocID := allocs[1].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Verify no new allocs were created - require.Equal(2, len(out)) + // Ensure a replacement alloc was placed. + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } - // Verify follow up eval was created for the failed alloc - alloc, err := h.State.AllocByID(ws, failedAllocID) - require.Nil(err) - require.NotEmpty(alloc.FollowupEvalID) + // Assert that we have the correct number of each alloc name + expected := map[string]int{ + "my-job.web[0]": 1, + "my-job.web[1]": 2, + "my-job.web[2]": 1, + } + actual := make(map[string]int, 3) + for _, alloc := range out { + actual[alloc.Name] += 1 + } + require.Equal(t, actual, expected) - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { - t.Fatalf("bad: %#v", h.CreateEvals) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - followupEval := h.CreateEvals[0] - require.Equal(now.Add(delayDuration), followupEval.WaitUntil) } -func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } +func TestServiceSched_JobRegister(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - maxRestartAttempts := 3 - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: maxRestartAttempts, - Interval: 30 * time.Minute, - Delay: 5 * time.Second, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.ClientStatus = structs.AllocClientStatusRunning - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - expectedNumAllocs := 3 - expectedNumReschedTrackers := 1 - - failedAllocId := allocs[1].ID - failedNodeID := allocs[1].NodeID - - assert := assert.New(t) - for i := 0; i < maxRestartAttempts; i++ { - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - noErr(t, err) - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that a new allocation got created with its restart tracker info - assert.Equal(expectedNumAllocs, len(out)) - - // Find the new alloc with ClientStatusPending - var pendingAllocs []*structs.Allocation - var prevFailedAlloc *structs.Allocation - - for _, alloc := range out { - if alloc.ClientStatus == structs.AllocClientStatusPending { - pendingAllocs = append(pendingAllocs, alloc) - } - if alloc.ID == failedAllocId { - prevFailedAlloc = alloc - } - } - assert.Equal(1, len(pendingAllocs)) - newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) - - // Verify the previous NodeID in the most recent reschedule event - reschedEvents := newAlloc.RescheduleTracker.Events - assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) - assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) - - // Verify that the next alloc of the failed alloc is the newly rescheduled alloc - assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) - - // Mark this alloc as failed again - newAlloc.ClientStatus = structs.AllocClientStatusFailed - newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-12 * time.Second), - FinishedAt: now.Add(-10 * time.Second)}} - - failedAllocId = newAlloc.ID - failedNodeID = newAlloc.NodeID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expectedNumAllocs += 1 - expectedNumReschedTrackers += 1 - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process last eval again, should not reschedule - err := h.Process(NewServiceScheduler, eval) - assert.Nil(err) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Verify no new allocs were created because restart attempts were exhausted - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts -} + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } -// Tests that old reschedule attempts are pruned -func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - DelayFunction: "exponential", - MaxDelay: 1 * time.Hour, - Delay: 5 * time.Second, - Unlimited: true, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - now := time.Now() - // Mark allocations as failed with restart info - allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-15 * time.Minute)}} - allocs[1].ClientStatus = structs.AllocClientStatusFailed - - allocs[1].RescheduleTracker = &structs.RescheduleTracker{ - Events: []*structs.RescheduleEvent{ - {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), - PrevAllocID: uuid.Generate(), - PrevNodeID: uuid.Generate(), - Delay: 5 * time.Second, - }, - {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 10 * time.Second, - }, - {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 20 * time.Second, - }, - {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 40 * time.Second, - }, - {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 80 * time.Second, - }, - {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 160 * time.Second, - }, - }, - } - expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] - expectedDelay := 320 * time.Second - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the eval has no spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - // Verify that the new alloc copied the last 5 reschedule attempts - assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] - // Verify that the failed alloc ID is in the most recent reschedule event - assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) - // Verify that the delay value was captured correctly - assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) + // Ensure different ports were used. + used := make(map[int]map[string]struct{}) + for _, alloc := range out { + for _, resource := range alloc.TaskResources { + for _, port := range resource.Networks[0].DynamicPorts { + nodeMap, ok := used[port.Value] + if !ok { + nodeMap = make(map[string]struct{}) + used[port.Value] = nodeMap + } + if _, ok := nodeMap[alloc.NodeID]; ok { + t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) + } + nodeMap[alloc.NodeID] = struct{}{} + } + } + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } -// Tests that deployments with failed allocs result in placements as long as the -// deployment is running. -func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { - for _, failedDeployment := range []bool{false, true} { - t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { - h := NewHarness(t) - require := require.New(t) +func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + // Create some nodes - var nodes []*structs.Node for i := 0; i < 10; i++ { node := mock.Node() - nodes = append(nodes, node) noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Generate a fake job with allocations and a reschedule policy. + // Create a job job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - if failedDeployment { - deployment.Status = structs.DeploymentStatusFailed + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID - allocs = append(allocs, alloc) + // Ensure the plan allocated + plan := h.Plans[0] + planned := make(map[string]*structs.Allocation) + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + planned[alloc.ID] = alloc + } + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) } - // Mark one of the allocations as failed in the past - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", - StartedAt: time.Now().Add(-12 * time.Hour), - FinishedAt: time.Now().Add(-10 * time.Hour)}} - allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Update the job to force a rolling upgrade + updated := job.Copy() + updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 + noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) - // Create a mock evaluation - eval := &structs.Evaluation{ + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ Namespace: structs.DefaultNamespace, ID: uuid.Generate(), - Priority: 50, + Priority: job.Priority, TriggeredBy: structs.EvalTriggerNodeUpdate, JobID: job.ID, Status: structs.EvalStatusPending, } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) - - if failedDeployment { - // Verify no plan created - require.Len(h.Plans, 0) - } else { - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) + // Ensure we have created only one new allocation + // Ensure a single plan + if len(h1.Plans) != 1 { + t.Fatalf("bad: %#v", h1.Plans) + } + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 10 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocations were placed on the same node as the older + // ones + for _, new := range newPlanned { + if new.PreviousAllocation == "" { + t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) + } + + old, ok := planned[new.PreviousAllocation] + if !ok { + t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) + if new.NodeID != old.NodeID { + t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) } } }) } } -func TestBatchSched_Run_CompleteAlloc(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan as it should be a no-op - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestBatchSched_Run_FailedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - tgName := job.TaskGroups[0].Name - now := time.Now() - - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a job with count 2 and disk as 60GB so that only one allocation + // can fit + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + ID: uuid.Generate(), + } - // Ensure a replacement alloc was placed. - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] -func TestBatchSched_Run_LostAlloc(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.Job() - job.ID = "my-job" - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Desired = 3 - // Mark one as lost and then schedule - // [(0, run, running), (1, run, running), (1, stop, lost)] - - // Create two running allocations - var allocs []*structs.Allocation - for i := 0; i <= 1; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[1]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the eval has a blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated only one allocation + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure a replacement alloc was placed. - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Ensure only one allocation was placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Assert that we have the correct number of each alloc name - expected := map[string]int{ - "my-job.web[0]": 1, - "my-job.web[1]": 2, - "my-job.web[2]": 1, - } - actual := make(map[string]int, 3) - for _, alloc := range out { - actual[alloc.Name] += 1 + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - require.Equal(t, actual, expected) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { - h := NewHarness(t) - - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - tgName := job.TaskGroups[0].Name - now := time.Now() + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 1 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 1 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a successful alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a successful alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, }, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to rerun the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to rerun the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no replacement alloc was placed. - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure no replacement alloc was placed. + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This test checks that terminal allocations that receive an in-place updated // are not added to the plan func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to trigger the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to trigger the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans[0]) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans[0]) + } + }) } } // This test ensures that terminal jobs from older versions are ignored. func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - job2.Type = structs.JobTypeBatch - job2.Version++ - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - allocs = nil - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job2 - alloc.JobID = job2.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - }, - } - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + job2.Type = structs.JobTypeBatch + job2.Version++ + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + allocs = nil + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job2 + alloc.JobID = job2.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + }, + } + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) + // Ensure a plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + }) } } // This test asserts that an allocation from an old job that is running on a // drained node is cleaned up. func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a running alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create an update job - job2 := job.Copy() - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - job2.Version++ - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a running alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create an update job + job2 := job.Copy() + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + job2.Version++ + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - plan := h.Plans[0] + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan evicted 1 - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + plan := h.Plans[0] - // Ensure the plan places 1 - if len(plan.NodeAllocation[node2.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted 1 + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan places 1 + if len(plan.NodeAllocation[node2.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This test asserts that an allocation from a job that is complete on a // drained node is ignored up. func TestBatchSched_NodeDrain_Complete(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = make(map[string]*structs.TaskState) - alloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = make(map[string]*structs.TaskState) + alloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This is a slightly odd test but it ensures that we handle a scale down of a // task group's count and that it works even if all the allocs have the same // name. func TestBatchSched_ScaleDown_SameName(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a few running alloc - var allocs []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a few running alloc + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - plan := h.Plans[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan evicted 4 of the 5 - if len(plan.NodeUpdate[node.ID]) != 4 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + plan := h.Plans[0] + + // Ensure the plan evicted 4 of the 5 + if len(plan.NodeUpdate[node.ID]) != 4 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestGenericSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.Job() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - job1.TaskGroups[0].Count = 12 - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - if err := h1.Process(NewServiceScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.Job() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + job1.TaskGroups[0].Count = 12 + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - plan := h1.Plans[0] + // Process the evaluation + if err := h1.Process(NewServiceScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestServiceSched_NodeDrain_Sticky(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create an alloc on the draining node + alloc := mock.Alloc() + alloc.Name = "my-job.web[0]" + alloc.NodeID = node.ID + alloc.Job.TaskGroups[0].Count = 1 + alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: alloc.Job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Create an alloc on the draining node - alloc := mock.Alloc() - alloc.Name = "my-job.web[0]" - alloc.NodeID = node.ID - alloc.Job.TaskGroups[0].Count = 1 - alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: alloc.Job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't create any new allocations + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't create any new allocations - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is stopped, the scheduler properly cancels // an outstanding deployment. func TestServiceSched_CancelDeployment_Stopped(t *testing.T) { - h := NewHarness(t) - - // Generate a fake job - job := mock.Job() - job.JobModifyIndex = job.CreateIndex + 1 - job.ModifyIndex = job.CreateIndex + 1 - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment - d := mock.Deployment() - d.JobID = job.ID - d.JobCreateIndex = job.CreateIndex - d.JobModifyIndex = job.JobModifyIndex - 1 - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job + job := mock.Job() + job.JobModifyIndex = job.CreateIndex + 1 + job.ModifyIndex = job.CreateIndex + 1 + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a deployment + d := mock.Deployment() + d.JobID = job.ID + d.JobCreateIndex = job.CreateIndex + d.JobModifyIndex = job.JobModifyIndex - 1 + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) - } + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) + } + + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This test ensures that when a job is updated and had an old deployment, the scheduler properly cancels // the deployment. func TestServiceSched_CancelDeployment_NewerJob(t *testing.T) { - h := NewHarness(t) - - // Generate a fake job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment for an old version of the job - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Upsert again to bump job version - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to kick the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a deployment for an old version of the job + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Upsert again to bump job version + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to kick the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) - } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) + + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) + } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // Various table driven tests for carry forward // of past reschedule events func Test_updateRescheduleTracker(t *testing.T) { - t1 := time.Now().UTC() alloc := mock.Alloc() prevAlloc := mock.Alloc() @@ -4603,5 +4818,4 @@ func Test_updateRescheduleTracker(t *testing.T) { require.Equal(tc.expectedRescheduleEvents, alloc.RescheduleTracker.Events) }) } - } diff --git a/scheduler/system_sched_test.go b/scheduler/system_sched_test.go index 9461fd85eaa..2509697de93 100644 --- a/scheduler/system_sched_test.go +++ b/scheduler/system_sched_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -15,1870 +15,1964 @@ import ( "github.com/stretchr/testify/require" ) +// COMPAT 0.11: Currently, all the tests run for 2 cases: +// 1) Allow plan optimization +// 2) Not allowing plan optimization +// The code for not allowing plan optimizations is in place to allow for safer cluster upgrades, +// and backwards compatibility with the existing raft logs. The older code will be removed later, +// and these tests should then no longer include the testing for case (2) + func TestSystemSched_JobRegister(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure no allocations are queued + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) + } - // Ensure no allocations are queued - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - plan := h.Plans[0] - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + plan := h.Plans[0] + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Get an allocation and mark it as failed - alloc := planned[4].Copy() - alloc.ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Get an allocation and mark it as failed + alloc := planned[4].Copy() + alloc.ClientStatus = structs.AllocClientStatusFailed + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure we have created only one new allocation - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 1 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocation was placed on the same node as the older - // one - if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { - t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + // Ensure we have created only one new allocation + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 1 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocation was placed on the same node as the older + // one + if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { + t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + } + }) } } func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { - h := NewHarness(t) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create another job with a lot of disk resource ask so that it doesn't fit - // the node - job1 := mock.SystemJob() - job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create another job with a lot of disk resource ask so that it doesn't fit + // the node + job1 := mock.SystemJob() + job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Create a new harness to test the scheduling result for the second job - h1 := NewHarnessWithState(t, h.State) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a new harness to test the scheduling result for the second job + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) - noErr(t, err) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) + out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) + noErr(t, err) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + }) } } func TestSystemSched_ExhaustResources(t *testing.T) { - h := NewHarness(t) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create a service job which consumes most of the system resources - svcJob := mock.Job() - svcJob.TaskGroups[0].Count = 1 - svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: svcJob.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: svcJob.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create a service job which consumes most of the system resources + svcJob := mock.Job() + svcJob.TaskGroups[0].Count = 1 + svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: svcJob.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: svcJob.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a system job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Create a system job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // System scheduler will preempt the service job and would have placed eval1 - require := require.New(t) + // System scheduler will preempt the service job and would have placed eval1 + require := require.New(t) - newPlan := h.Plans[1] - require.Len(newPlan.NodeAllocation, 1) - require.Len(newPlan.NodePreemptions, 1) + newPlan := h.Plans[1] + require.Len(newPlan.NodeAllocation, 1) + require.Len(newPlan.NodePreemptions, 1) - for _, allocList := range newPlan.NodeAllocation { - require.Len(allocList, 1) - require.Equal(job.ID, allocList[0].JobID) - } + for _, allocList := range newPlan.NodeAllocation { + require.Len(allocList, 1) + require.Equal(job.ID, allocList[0].JobID) + } - for _, allocList := range newPlan.NodePreemptions { - require.Len(allocList, 1) - require.Equal(svcJob.ID, allocList[0].JobID) - } - // Ensure that we have no queued allocations on the second eval - queued := h.Evals[1].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + for _, allocList := range newPlan.NodePreemptions { + require.Len(allocList, 1) + alloc, err := h.State.AllocByID(nil, allocList[0].ID) + noErr(t, err) + require.Equal(svcJob.ID, alloc.JobID) + } + // Ensure that we have no queued allocations on the second eval + queued := h.Evals[1].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestSystemSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - if i < 9 { - node.NodeClass = "foo" - } else { - node.NodeClass = "bar" - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + if i < 9 { + node.NodeClass = "foo" + } else { + node.NodeClass = "bar" + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job constraining on node class - job := mock.SystemJob() - fooConstraint := &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "foo", - Operand: "==", - } - job.Constraints = append(job.Constraints, fooConstraint) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job constraining on node class + job := mock.SystemJob() + fooConstraint := &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "foo", + Operand: "==", + } + job.Constraints = append(job.Constraints, fooConstraint) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 9 { - t.Fatalf("bad: %#v %d", planned, len(planned)) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 9 { + t.Fatalf("bad: %#v %d", planned, len(planned)) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 9 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 9 { + t.Fatalf("bad: %#v", out) + } - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - expected := &structs.DesiredUpdates{Place: 9} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + expected := &structs.DesiredUpdates{Place: 9} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + + } + }) } } func TestSystemSched_JobRegister_AddNode(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a new node. - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a new node. + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan had no node updates - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Log(len(update)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan had no node updates + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Log(len(update)) + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated on the new node - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated on the new node + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure it allocated on the right node - if _, ok := plan.NodeAllocation[node.ID]; !ok { - t.Fatalf("allocated on wrong node: %#v", plan) - } + // Ensure it allocated on the right node + if _, ok := plan.NodeAllocation[node.ID]; !ok { + t.Fatalf("allocated on wrong node: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 11 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 11 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) - - // Create NO nodes - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create NO nodes + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan as this should be a no-op. - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan as this should be a no-op. + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobModify(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = "my-job.web[0]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = "my-job.web[0]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - job2.Update = structs.UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 5, - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + job2.Update = structs.UpdateStrategy{ + Stagger: 30 * time.Second, + MaxParallel: 5, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a follow up eval was created - eval = h.Evals[0] - if eval.NextEval == "" { - t.Fatalf("missing next eval") - } + // Ensure a follow up eval was created + eval = h.Evals[0] + if eval.NextEval == "" { + t.Fatalf("missing next eval") + } - // Check for create - if len(h.CreateEvals) == 0 { - t.Fatalf("missing created eval") - } - create := h.CreateEvals[0] - if eval.NextEval != create.ID { - t.Fatalf("ID mismatch") - } - if create.PreviousEval != eval.ID { - t.Fatalf("missing previous eval") - } + // Check for create + if len(h.CreateEvals) == 0 { + t.Fatalf("missing created eval") + } + create := h.CreateEvals[0] + if eval.NextEval != create.ID { + t.Fatalf("ID mismatch") + } + if create.PreviousEval != eval.ID { + t.Fatalf("missing previous eval") + } - if create.TriggeredBy != structs.EvalTriggerRollingUpdate { - t.Fatalf("bad: %#v", create) + if create.TriggeredBy != structs.EvalTriggerRollingUpdate { + t.Fatalf("bad: %#v", create) + } + }) } } func TestSystemSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } + } + }) } } func TestSystemSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeDown(t *testing.T) { - h := NewHarness(t) - - // Register a down node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a down node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && - p.ClientStatus != structs.AllocClientStatusLost { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the allocations is stopped + if p := planned[0]; p.DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the allocation is marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - expected := []string{alloc.ID} + // Ensure that the allocation is marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + expected := []string{alloc.ID} - if !reflect.DeepEqual(lostAllocs, expected) { - t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) + if !reflect.DeepEqual(lostAllocs, expected) { + t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Log(len(planned)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Log(len(planned)) + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) + }) } - - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) } // This test ensures that the scheduler doesn't increment the queued allocation // count for a task group when allocations can't be created on currently // available nodes because of constrain mismatches. func TestSystemSched_Queued_With_Constraints(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - node.Attributes["kernel.name"] = "darwin" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a system job which can't be placed on the node - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + node.Attributes["kernel.name"] = "darwin" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a system job which can't be placed on the node + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } + }) } } func TestSystemSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.SystemJob() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Insert two more nodes - for i := 0; i < 2; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.SystemJob() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Insert two more nodes + for i := 0; i < 2; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - plan := h1.Plans[0] + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestSystemSched_PlanWithDrainedNode(t *testing.T) { - h := NewHarness(t) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.Drain = true - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.Drain = true + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create an allocation on each node + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + alloc.TaskGroup = "web" + + alloc2 := mock.Alloc() + alloc2.Job = job + alloc2.JobID = job.ID + alloc2.NodeID = node2.ID + alloc2.Name = "my-job.web2[0]" + alloc2.TaskGroup = "web2" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create an allocation on each node - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - alloc.TaskGroup = "web" - - alloc2 := mock.Alloc() - alloc2.Job = job - alloc2.JobID = job.ID - alloc2.NodeID = node2.ID - alloc2.Name = "my-job.web2[0]" - alloc2.TaskGroup = "web2" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan evicted the alloc on the failed node + planned := plan.NodeUpdate[node.ID] + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted the alloc on the failed node - planned := plan.NodeUpdate[node.ID] - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't place + if len(plan.NodeAllocation) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't place - if len(plan.NodeAllocation) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { - h := NewHarness(t) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + qa := h.Evals[0].QueuedAllocations + if qa["web"] != 0 || qa["web2"] != 0 { + t.Fatalf("bad queued allocations %#v", qa) + } - qa := h.Evals[0].QueuedAllocations - if qa["web"] != 0 || qa["web2"] != 0 { - t.Fatalf("bad queued allocations %#v", qa) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_Preemption(t *testing.T) { - h := NewHarness(t) - - // Create nodes - var nodes []*structs.Node - for i := 0; i < 2; i++ { - node := mock.Node() - //TODO(preetha): remove in 0.11 - node.Resources = &structs.Resources{ - CPU: 3072, - MemoryMB: 5034, - DiskMB: 20 * 1024, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - node.NodeResources = &structs.NodeResources{ - Cpu: structs.NodeCpuResources{ - CpuShares: 3072, - }, - Memory: structs.NodeMemoryResources{ - MemoryMB: 5034, - }, - Disk: structs.NodeDiskResources{ - DiskMB: 20 * 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - nodes = append(nodes, node) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create nodes + var nodes []*structs.Node + for i := 0; i < 2; i++ { + node := mock.Node() + //TODO(preetha): remove in 0.11 + node.Resources = &structs.Resources{ + CPU: 3072, + MemoryMB: 5034, + DiskMB: 20 * 1024, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + node.NodeResources = &structs.NodeResources{ + Cpu: structs.NodeCpuResources{ + CpuShares: 3072, + }, + Memory: structs.NodeMemoryResources{ + MemoryMB: 5034, + }, + Disk: structs.NodeDiskResources{ + DiskMB: 20 * 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + nodes = append(nodes, node) + } - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create some low priority batch jobs and allocations for them - // One job uses a reserved port - job1 := mock.BatchJob() - job1.Type = structs.JobTypeBatch - job1.Priority = 20 - job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - ReservedPorts: []structs.Port{ + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create some low priority batch jobs and allocations for them + // One job uses a reserved port + job1 := mock.BatchJob() + job1.Type = structs.JobTypeBatch + job1.Priority = 20 + job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ { - Label: "web", - Value: 80, + MBits: 200, + ReservedPorts: []structs.Port{ + { + Label: "web", + Value: 80, + }, + }, }, }, - }, - }, - } + } - alloc1 := mock.Alloc() - alloc1.Job = job1 - alloc1.JobID = job1.ID - alloc1.NodeID = nodes[0].ID - alloc1.Name = "my-job[0]" - alloc1.TaskGroup = job1.TaskGroups[0].Name - alloc1.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, + alloc1 := mock.Alloc() + alloc1.Job = job1 + alloc1.JobID = job1.ID + alloc1.NodeID = nodes[0].ID + alloc1.Name = "my-job[0]" + alloc1.TaskGroup = job1.TaskGroups[0].Name + alloc1.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 200, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + job2 := mock.BatchJob() + job2.Type = structs.JobTypeBatch + job2.Priority = 20 + job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 200, + MBits: 200, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - job2 := mock.BatchJob() - job2.Type = structs.JobTypeBatch - job2.Priority = 20 - job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - }, - }, - } + } - alloc2 := mock.Alloc() - alloc2.Job = job2 - alloc2.JobID = job2.ID - alloc2.NodeID = nodes[0].ID - alloc2.Name = "my-job[2]" - alloc2.TaskGroup = job2.TaskGroups[0].Name - alloc2.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, + alloc2 := mock.Alloc() + alloc2.Job = job2 + alloc2.JobID = job2.ID + alloc2.NodeID = nodes[0].ID + alloc2.Name = "my-job[2]" + alloc2.TaskGroup = job2.TaskGroups[0].Name + alloc2.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + MBits: 200, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + job3 := mock.Job() + job3.Type = structs.JobTypeBatch + job3.Priority = 40 + job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, Networks: []*structs.NetworkResource{ { Device: "eth0", - IP: "192.168.0.100", - MBits: 200, + MBits: 400, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - job3 := mock.Job() - job3.Type = structs.JobTypeBatch - job3.Priority = 40 - job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - MBits: 400, - }, - }, - } + } - alloc3 := mock.Alloc() - alloc3.Job = job3 - alloc3.JobID = job3.ID - alloc3.NodeID = nodes[0].ID - alloc3.Name = "my-job[0]" - alloc3.TaskGroup = job3.TaskGroups[0].Name - alloc3.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, + alloc3 := mock.Alloc() + alloc3.Job = job3 + alloc3.JobID = job3.ID + alloc3.NodeID = nodes[0].ID + alloc3.Name = "my-job[0]" + alloc3.TaskGroup = job3.TaskGroups[0].Name + alloc3.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 25, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 400, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 25, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) + + // Create a high priority job and allocs for it + // These allocs should not be preempted + + job4 := mock.BatchJob() + job4.Type = structs.JobTypeBatch + job4.Priority = 100 + job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 400, + MBits: 100, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) - - // Create a high priority job and allocs for it - // These allocs should not be preempted - - job4 := mock.BatchJob() - job4.Type = structs.JobTypeBatch - job4.Priority = 100 - job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - MBits: 100, - }, - }, - } + } - alloc4 := mock.Alloc() - alloc4.Job = job4 - alloc4.JobID = job4.ID - alloc4.NodeID = nodes[0].ID - alloc4.Name = "my-job4[0]" - alloc4.TaskGroup = job4.TaskGroups[0].Name - alloc4.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, + alloc4 := mock.Alloc() + alloc4.Job = job4 + alloc4.JobID = job4.ID + alloc4.NodeID = nodes[0].ID + alloc4.Name = "my-job4[0]" + alloc4.TaskGroup = job4.TaskGroups[0].Name + alloc4.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 2048, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 100, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2048, + Shared: structs.AllocatedSharedResources{ + DiskMB: 2 * 1024, }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) + + // Create a system job such that it would need to preempt both allocs to succeed + job := mock.SystemJob() + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1948, + MemoryMB: 256, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 100, + MBits: 800, + DynamicPorts: []structs.Port{{Label: "http"}}, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 2 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) - - // Create a system job such that it would need to preempt both allocs to succeed - job := mock.SystemJob() - job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1948, - MemoryMB: 256, - Networks: []*structs.NetworkResource{ - { - MBits: 800, - DynamicPorts: []structs.Port{{Label: "http"}}, - }, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - require := require.New(t) - require.Nil(err) - - // Ensure a single plan - require.Equal(1, len(h.Plans)) - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - require.Nil(plan.Annotations) - - // Ensure the plan allocated on both nodes - var planned []*structs.Allocation - preemptingAllocId := "" - require.Equal(2, len(plan.NodeAllocation)) - - // The alloc that got placed on node 1 is the preemptor - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - for _, alloc := range allocList { - if alloc.NodeID == nodes[0].ID { - preemptingAllocId = alloc.ID } - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + require := require.New(t) + require.Nil(err) + + // Ensure a single plan + require.Equal(1, len(h.Plans)) + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + require.Nil(plan.Annotations) + + // Ensure the plan allocated on both nodes + var planned []*structs.Allocation + preemptingAllocId := "" + require.Equal(2, len(plan.NodeAllocation)) + + // The alloc that got placed on node 1 is the preemptor + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + for _, alloc := range allocList { + if alloc.NodeID == nodes[0].ID { + preemptingAllocId = alloc.ID + } + } + } - // Ensure all allocations placed - require.Equal(2, len(out)) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Verify that one node has preempted allocs - require.NotNil(plan.NodePreemptions[nodes[0].ID]) - preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + // Ensure all allocations placed + require.Equal(2, len(out)) - // Verify that three jobs have preempted allocs - require.Equal(3, len(preemptedAllocs)) + // Verify that one node has preempted allocs + require.NotNil(plan.NodePreemptions[nodes[0].ID]) + preemptedAllocs := plan.NodePreemptions[nodes[0].ID] - expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} + // Verify that three jobs have preempted allocs + require.Equal(3, len(preemptedAllocs)) - // We expect job1, job2 and job3 to have preempted allocations - // job4 should not have any allocs preempted - for _, alloc := range preemptedAllocs { - require.Contains(expectedPreemptedJobIDs, alloc.JobID) - } - // Look up the preempted allocs by job ID - ws = memdb.NewWatchSet() - - for _, jobId := range expectedPreemptedJobIDs { - out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) - noErr(t, err) - for _, alloc := range out { - require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) - require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) - } - } + expectedPreemptedAllocIDs := []string{alloc1.ID, alloc2.ID, alloc3.ID} - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // We expect job1, job2 and job3 to have preempted allocations + // job4 should not have any allocs preempted + for _, alloc := range preemptedAllocs { + require.Contains(expectedPreemptedAllocIDs, alloc.ID) + } + // Look up the preempted allocs by job ID + ws = memdb.NewWatchSet() + + for _, allocID := range expectedPreemptedAllocIDs { + evictedAlloc, err := h.State.AllocByID(ws, allocID) + noErr(t, err) + require.Equal(structs.AllocDesiredStatusEvict, evictedAlloc.DesiredStatus) + require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), evictedAlloc.DesiredDescription) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } diff --git a/scheduler/testing.go b/scheduler/testing.go index 61fa7f79c61..dfe4f728639 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -5,9 +5,9 @@ import ( "sync" "time" - testing "github.com/mitchellh/go-testing-interface" + "github.com/mitchellh/go-testing-interface" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" @@ -53,26 +53,30 @@ type Harness struct { nextIndex uint64 nextIndexLock sync.Mutex + + allowPlanOptimization bool } // NewHarness is used to make a new testing harness -func NewHarness(t testing.T) *Harness { +func NewHarness(t testing.T, allowPlanOptimization bool) *Harness { state := state.TestStateStore(t) h := &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } return h } // NewHarnessWithState creates a new harness with the given state for testing // purposes. -func NewHarnessWithState(t testing.T, state *state.StateStore) *Harness { +func NewHarnessWithState(t testing.T, state *state.StateStore, allowPlanOptimization bool) *Harness { return &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } } @@ -101,22 +105,17 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er result.AllocIndex = index // Flatten evicts and allocs - var allocs []*structs.Allocation + now := time.Now().UTC().UnixNano() + allocsStopped := make([]*structs.Allocation, 0, len(result.NodeUpdate)) for _, updateList := range plan.NodeUpdate { - allocs = append(allocs, updateList...) - } - for _, allocList := range plan.NodeAllocation { - allocs = append(allocs, allocList...) + allocsStopped = append(allocsStopped, updateList...) } - // Set the time the alloc was applied for the first time. This can be used - // to approximate the scheduling time. - now := time.Now().UTC().UnixNano() - for _, alloc := range allocs { - if alloc.CreateTime == 0 { - alloc.CreateTime = now - } + allocsUpdated := make([]*structs.Allocation, 0, len(result.NodeAllocation)) + for _, allocList := range plan.NodeAllocation { + allocsUpdated = append(allocsUpdated, allocList...) } + updateCreateTimestamp(allocsUpdated, now) // Set modify time for preempted allocs and flatten them var preemptedAllocs []*structs.Allocation @@ -130,8 +129,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: plan.Job, - Alloc: allocs, + Job: plan.Job, }, Deployment: plan.Deployment, DeploymentUpdates: plan.DeploymentUpdates, @@ -139,11 +137,33 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er NodePreemptions: preemptedAllocs, } + if h.allowPlanOptimization { + req.AllocsStopped = allocsStopped + req.AllocsUpdated = allocsUpdated + } else { + // Deprecated: Handles unoptimized log format + var allocs []*structs.Allocation + allocs = append(allocs, allocsStopped...) + allocs = append(allocs, allocsUpdated...) + updateCreateTimestamp(allocs, now) + req.Alloc = allocs + } + // Apply the full plan err := h.State.UpsertPlanResults(index, &req) return result, nil, err } +func updateCreateTimestamp(allocations []*structs.Allocation, now int64) { + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + for _, alloc := range allocations { + if alloc.CreateTime == 0 { + alloc.CreateTime = now + } + } +} + func (h *Harness) UpdateEval(eval *structs.Evaluation) error { // Ensure sequential plan application h.planLock.Lock() @@ -214,15 +234,15 @@ func (h *Harness) Snapshot() State { // Scheduler is used to return a new scheduler from // a snapshot of current state using the harness for planning. -func (h *Harness) Scheduler(factory Factory) Scheduler { +func (h *Harness) Scheduler(factory Factory, allowPlanOptimization bool) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h, false) + return factory(logger, h.Snapshot(), h, allowPlanOptimization) } // Process is used to process an evaluation given a factory // function to create the scheduler func (h *Harness) Process(factory Factory, eval *structs.Evaluation) error { - sched := h.Scheduler(factory) + sched := h.Scheduler(factory, h.allowPlanOptimization) return sched.Process(eval) } diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 08f5812aac6..13864b43214 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -621,7 +621,7 @@ func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { } func TestSetStatus(t *testing.T) { - h := NewHarness(t) + h := NewHarness(t, true) logger := testlog.HCLogger(t) eval := mock.Eval() status := "a" @@ -640,7 +640,7 @@ func TestSetStatus(t *testing.T) { } // Test next evals - h = NewHarness(t) + h = NewHarness(t, true) next := mock.Eval() if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -656,7 +656,7 @@ func TestSetStatus(t *testing.T) { } // Test blocked evals - h = NewHarness(t) + h = NewHarness(t, true) blocked := mock.Eval() if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -672,7 +672,7 @@ func TestSetStatus(t *testing.T) { } // Test metrics - h = NewHarness(t) + h = NewHarness(t, true) metrics := map[string]*structs.AllocMetric{"foo": nil} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -688,7 +688,7 @@ func TestSetStatus(t *testing.T) { } // Test queued allocations - h = NewHarness(t) + h = NewHarness(t, true) queuedAllocs := map[string]int{"web": 1} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { @@ -704,7 +704,7 @@ func TestSetStatus(t *testing.T) { t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) } - h = NewHarness(t) + h = NewHarness(t, true) dID := uuid.Generate() if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { t.Fatalf("setStatus() failed: %v", err) From 3ab267c0a169c4d3719957601943318da5209d7f Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Fri, 8 Mar 2019 03:18:56 -0800 Subject: [PATCH 6/7] Compat tags --- nomad/fsm.go | 2 +- nomad/job_endpoint.go | 2 +- nomad/plan_apply.go | 40 ++++++++++++++++++++------------- nomad/plan_apply_test.go | 2 +- nomad/structs/structs.go | 1 + nomad/util.go | 6 ++--- scheduler/generic_sched_test.go | 2 +- scheduler/testing.go | 2 +- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/nomad/fsm.go b/nomad/fsm.go index fc3cebad1b4..c54a40d86da 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -1427,7 +1427,7 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanDenormalization, true) + allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanNormalization, true) sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) if err != nil { return err diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index f6a0ace535c..962dc4f9af8 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1213,7 +1213,7 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanDenormalization, true) + allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanNormalization, true) sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) if err != nil { return err diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index dc991323288..777b5e8cc6c 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -163,7 +163,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap preemptedJobIDs := make(map[structs.NamespacedID]struct{}) now := time.Now().UTC().UnixNano() - if ServersMeetMinimumVersion(p.Members(), MinVersionPlanDenormalization, true) { + if ServersMeetMinimumVersion(p.Members(), MinVersionPlanNormalization, true) { // Initialize the allocs request using the new optimized log entry format. // Determine the minimum number of updates, could be more if there // are multiple updates per node @@ -172,12 +172,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap for _, updateList := range result.NodeUpdate { for _, stoppedAlloc := range updateList { - req.AllocsStopped = append(req.AllocsStopped, &structs.Allocation{ - ID: stoppedAlloc.ID, - DesiredDescription: stoppedAlloc.DesiredDescription, - ClientStatus: stoppedAlloc.ClientStatus, - ModifyTime: now, - }) + req.AllocsStopped = append(req.AllocsStopped, normalizeStoppedAlloc(stoppedAlloc, now)) } } @@ -191,18 +186,14 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap for _, preemptions := range result.NodePreemptions { for _, preemptedAlloc := range preemptions { - req.NodePreemptions = append(req.NodePreemptions, &structs.Allocation{ - ID: preemptedAlloc.ID, - PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, - ModifyTime: now, - }) + req.NodePreemptions = append(req.NodePreemptions, normalizePreemptedAlloc(preemptedAlloc, now)) // Gather jobids to create follow up evals appendNamespacedJobID(preemptedJobIDs, preemptedAlloc) } } } else { - // Deprecated: This code path is deprecated and will only be used to support + // COMPAT 0.11: This branch is deprecated and will only be used to support // application of older log entries. Expected to be removed in a future version. // Determine the minimum number of updates, could be more if there @@ -270,6 +261,23 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap return future, nil } +func normalizePreemptedAlloc(preemptedAlloc *structs.Allocation, now int64) *structs.Allocation { + return &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, + ModifyTime: now, + } +} + +func normalizeStoppedAlloc(stoppedAlloc *structs.Allocation, now int64) *structs.Allocation { + return &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: stoppedAlloc.DesiredDescription, + ClientStatus: stoppedAlloc.ClientStatus, + ModifyTime: now, + } +} + func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *structs.Allocation) { id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} if _, ok := jobIDs[id]; !ok { @@ -318,11 +326,13 @@ func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, func evaluatePlan(pool *EvaluatePool, snap *state.StateSnapshot, plan *structs.Plan, logger log.Logger) (*structs.PlanResult, error) { defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now()) - err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, plan.Job) + // Denormalize without the job + err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, nil) if err != nil { return nil, err } - err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, plan.Job) + // Denormalize without the job + err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, nil) if err != nil { return nil, err } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 619b7b9071d..538a7183aa4 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -63,7 +63,7 @@ func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { } } -// Deprecated: Tests the older unoptimized code path for applyPlan +// COMPAT 0.11: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { t.Parallel() s1 := TestServer(t, nil) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index be089165a74..b56c43c19e4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -667,6 +667,7 @@ type ApplyPlanResultsRequest struct { // to cause evictions or to assign new allocations. Both can be done // within a single transaction type AllocUpdateRequest struct { + // COMPAT 0.11 // Alloc is the list of new allocations to assign // Deprecated: Replaced with two separate slices, one containing stopped allocations // and another containing updated allocations diff --git a/nomad/util.go b/nomad/util.go index 8a6ff407f73..74dc41e04ec 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -14,10 +14,10 @@ import ( "github.com/hashicorp/serf/serf" ) -// MinVersionPlanDenormalization is the minimum version to support the -// denormalization of Plan in SubmitPlan, and the raft log entry committed +// MinVersionPlanNormalization is the minimum version to support the +// normalization of Plan in SubmitPlan, and the denormalization raft log entry committed // in ApplyPlanResultsRequest -var MinVersionPlanDenormalization = version.Must(version.NewVersion("0.9.1")) +var MinVersionPlanNormalization = version.Must(version.NewVersion("0.9.1")) // ensurePath is used to make sure a path exists func ensurePath(path string, dir bool) error { diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 223d41d5bfe..81cfbd2ec6b 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -554,7 +554,7 @@ func TestServiceSched_EvenSpread(t *testing.T) { func TestServiceSched_JobRegister_Annotate(t *testing.T) { for _, allowPlanOptimization := range []bool{true, false} { t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) // Create some nodes for i := 0; i < 10; i++ { diff --git a/scheduler/testing.go b/scheduler/testing.go index dfe4f728639..6eb43794572 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -141,7 +141,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er req.AllocsStopped = allocsStopped req.AllocsUpdated = allocsUpdated } else { - // Deprecated: Handles unoptimized log format + // COMPAT 0.11: Handles unoptimized log format var allocs []*structs.Allocation allocs = append(allocs, allocsStopped...) allocs = append(allocs, allocsUpdated...) From c242adea8786e7541c8c108c342026730b028ff4 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Fri, 8 Mar 2019 04:48:12 -0800 Subject: [PATCH 7/7] Remove allowPlanOptimization from schedulers --- nomad/fsm.go | 9 +- nomad/fsm_test.go | 2 +- nomad/job_endpoint.go | 3 +- nomad/plan_apply_test.go | 10 +- nomad/plan_normalization_test.go | 63 + nomad/server.go | 4 +- nomad/state/state_store_test.go | 10 +- nomad/structs/structs.go | 32 +- nomad/structs/structs_test.go | 55 +- nomad/worker.go | 8 +- nomad/worker_test.go | 18 +- scheduler/generic_sched.go | 32 +- scheduler/generic_sched_test.go | 7886 +++++++++++++++--------------- scheduler/scheduler.go | 6 +- scheduler/system_sched.go | 17 +- scheduler/system_sched_test.go | 3322 ++++++------- scheduler/testing.go | 36 +- scheduler/util_test.go | 12 +- 18 files changed, 5605 insertions(+), 5920 deletions(-) create mode 100644 nomad/plan_normalization_test.go diff --git a/nomad/fsm.go b/nomad/fsm.go index c54a40d86da..3c91b8f5c69 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -86,9 +86,6 @@ type nomadFSM struct { // new state store). Everything internal here is synchronized by the // Raft side, so doesn't need to lock this. stateLock sync.RWMutex - - // Reference to the server the FSM is running on - server *Server } // nomadSnapshot is used to provide a snapshot of the current @@ -124,7 +121,7 @@ type FSMConfig struct { } // NewFSMPath is used to construct a new FSM with a blank state -func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { +func NewFSM(config *FSMConfig) (*nomadFSM, error) { // Create a state store sconfig := &state.StateStoreConfig{ Logger: config.Logger, @@ -145,7 +142,6 @@ func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { timetable: NewTimeTable(timeTableGranularity, timeTableLimit), enterpriseAppliers: make(map[structs.MessageType]LogApplier, 8), enterpriseRestorers: make(map[SnapshotType]SnapshotRestorer, 8), - server: server, } // Register all the log applier functions @@ -1427,8 +1423,7 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanNormalization, true) - sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) + sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner) if err != nil { return err } diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 06177a0831d..0c3a846a85f 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -56,7 +56,7 @@ func testFSM(t *testing.T) *nomadFSM { Logger: logger, Region: "global", } - fsm, err := NewFSM(fsmConfig, TestServer(t, nil)) + fsm, err := NewFSM(fsmConfig) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 962dc4f9af8..132c14f58ce 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1213,8 +1213,7 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanNormalization, true) - sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) + sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner) if err != nil { return err } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 538a7183aa4..f58f95bc1cf 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -265,13 +265,13 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { alloc := mock.Alloc() stoppedAlloc := mock.Alloc() stoppedAllocDiff := &structs.Allocation{ - ID: stoppedAlloc.ID, + ID: stoppedAlloc.ID, DesiredDescription: "Desired Description", - ClientStatus: structs.AllocClientStatusLost, + ClientStatus: structs.AllocClientStatusLost, } preemptedAlloc := mock.Alloc() preemptedAllocDiff := &structs.Allocation{ - ID: preemptedAlloc.ID, + ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) @@ -356,7 +356,7 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { assert.NotNil(updatedPreemptedAlloc) assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) assert.Equal(updatedPreemptedAlloc.DesiredDescription, - "Preempted by alloc ID " + preemptedAllocDiff.PreemptedByAllocation) + "Preempted by alloc ID "+preemptedAllocDiff.PreemptedByAllocation) assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) // Lookup the new deployment diff --git a/nomad/plan_normalization_test.go b/nomad/plan_normalization_test.go new file mode 100644 index 00000000000..ab5f5b5caa5 --- /dev/null +++ b/nomad/plan_normalization_test.go @@ -0,0 +1,63 @@ +package nomad + +import ( + "bytes" + "testing" + "time" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/assert" + "github.com/ugorji/go/codec" +) + +// Whenever this test is changed, care should be taken to ensure the older msgpack size +// is recalculated when new fields are introduced in ApplyPlanResultsRequest +func TestPlanNormalize(t *testing.T) { + // This size was calculated using the older ApplyPlanResultsRequest format, in which allocations + // didn't use OmitEmpty and only the job was normalized in the stopped and preempted allocs. + // The newer format uses OmitEmpty and uses a minimal set of fields for the diff of the + // stopped and preempted allocs. The file for the older format hasn't been checked in, because + // it's not a good idea to check-in a 20mb file to the git repo. + unoptimizedLogSize := 19460168 + + numUpdatedAllocs := 10000 + numStoppedAllocs := 8000 + numPreemptedAllocs := 2000 + mockAlloc := mock.Alloc() + mockAlloc.Job = nil + + mockUpdatedAllocSlice := make([]*structs.Allocation, numUpdatedAllocs) + for i := 0; i < numUpdatedAllocs; i++ { + mockUpdatedAllocSlice = append(mockUpdatedAllocSlice, mockAlloc) + } + + now := time.Now().UTC().UnixNano() + mockStoppedAllocSlice := make([]*structs.Allocation, numStoppedAllocs) + for i := 0; i < numStoppedAllocs; i++ { + mockStoppedAllocSlice = append(mockStoppedAllocSlice, normalizeStoppedAlloc(mockAlloc, now)) + } + + mockPreemptionAllocSlice := make([]*structs.Allocation, numPreemptedAllocs) + for i := 0; i < numPreemptedAllocs; i++ { + mockPreemptionAllocSlice = append(mockPreemptionAllocSlice, normalizePreemptedAlloc(mockAlloc, now)) + } + + // Create a plan result + applyPlanLogEntry := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + AllocsUpdated: mockUpdatedAllocSlice, + AllocsStopped: mockStoppedAllocSlice, + }, + NodePreemptions: mockPreemptionAllocSlice, + } + + handle := structs.MsgpackHandle + var buf bytes.Buffer + if err := codec.NewEncoder(&buf, handle).Encode(applyPlanLogEntry); err != nil { + t.Fatalf("Encoding failed: %v", err) + } + + optimizedLogSize := buf.Len() + assert.True(t, float64(optimizedLogSize)/float64(unoptimizedLogSize) < 0.62) +} diff --git a/nomad/server.go b/nomad/server.go index 23142580d96..89da7a9e4b9 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1078,7 +1078,7 @@ func (s *Server) setupRaft() error { Region: s.Region(), } var err error - s.fsm, err = NewFSM(fsmConfig, s) + s.fsm, err = NewFSM(fsmConfig) if err != nil { return err } @@ -1173,7 +1173,7 @@ func (s *Server) setupRaft() error { if err != nil { return fmt.Errorf("recovery failed to parse peers.json: %v", err) } - tmpFsm, err := NewFSM(fsmConfig, s) + tmpFsm, err := NewFSM(fsmConfig) if err != nil { return fmt.Errorf("recovery failed to make temp FSM: %v", err) } diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index af248a55729..0e0539b48d7 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -153,14 +153,14 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { stoppedAlloc := mock.Alloc() stoppedAlloc.Job = job stoppedAllocDiff := &structs.Allocation{ - ID: stoppedAlloc.ID, + ID: stoppedAlloc.ID, DesiredDescription: "desired desc", - ClientStatus: structs.AllocClientStatusLost, + ClientStatus: structs.AllocClientStatusLost, } preemptedAlloc := mock.Alloc() preemptedAlloc.Job = job preemptedAllocDiff := &structs.Allocation{ - ID: preemptedAlloc.ID, + ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } @@ -185,9 +185,9 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { AllocUpdateRequest: structs.AllocUpdateRequest{ AllocsUpdated: []*structs.Allocation{alloc}, AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, - Job: job, + Job: job, }, - EvalID: eval.ID, + EvalID: eval.ID, NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, } assert := assert.New(t) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index b56c43c19e4..76056cff129 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8213,7 +8213,7 @@ func (e *Evaluation) ShouldBlock() bool { // MakePlan is used to make a plan from the given evaluation // for a given Job -func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { +func (e *Evaluation) MakePlan(j *Job) *Plan { p := &Plan{ EvalID: e.ID, Priority: e.Priority, @@ -8221,7 +8221,6 @@ func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { NodeUpdate: make(map[string][]*Allocation), NodeAllocation: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), - NormalizeAllocs: allowPlanOptimization, } if j != nil { p.AllAtOnce = j.AllAtOnce @@ -8343,9 +8342,6 @@ type Plan struct { // lower priority jobs that are preempted. Preempted allocations are marked // as evicted. NodePreemptions map[string][]*Allocation - - // Indicates whether allocs are normalized in the Plan - NormalizeAllocs bool `codec:"-"` } // AppendStoppedAlloc marks an allocation to be stopped. The clientStatus of the @@ -8439,23 +8435,21 @@ func (p *Plan) IsNoOp() bool { } func (p *Plan) NormalizeAllocations() { - if p.NormalizeAllocs { - for _, allocs := range p.NodeUpdate { - for i, alloc := range allocs { - allocs[i] = &Allocation{ - ID: alloc.ID, - DesiredDescription: alloc.DesiredDescription, - ClientStatus: alloc.ClientStatus, - } + for _, allocs := range p.NodeUpdate { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + DesiredDescription: alloc.DesiredDescription, + ClientStatus: alloc.ClientStatus, } } + } - for _, allocs := range p.NodePreemptions { - for i, alloc := range allocs { - allocs[i] = &Allocation{ - ID: alloc.ID, - PreemptedByAllocation: alloc.PreemptedByAllocation, - } + for _, allocs := range p.NodePreemptions { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: alloc.PreemptedByAllocation, } } } diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 2cd3922aebf..e574f51dc58 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -2842,13 +2842,12 @@ func TestTaskArtifact_Validate_Checksum(t *testing.T) { } } -func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { +func TestPlan_NormalizeAllocations(t *testing.T) { t.Parallel() plan := &Plan{ - NodeUpdate: make(map[string][]*Allocation), + NodeUpdate: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), } - plan.NormalizeAllocs = true stoppedAlloc := MockAlloc() desiredDesc := "Desired desc" plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) @@ -2873,45 +2872,6 @@ func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) } -func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsFalse(t *testing.T) { - t.Parallel() - plan := &Plan{ - NodeUpdate: make(map[string][]*Allocation), - NodePreemptions: make(map[string][]*Allocation), - } - plan.NormalizeAllocs = false - stoppedAlloc := MockAlloc() - desiredDesc := "Desired desc" - plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) - preemptedAlloc := MockAlloc() - preemptingAllocID := uuid.Generate() - plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) - - plan.NormalizeAllocations() - - actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] - expectedStoppedAlloc := new(Allocation) - *expectedStoppedAlloc = *stoppedAlloc - expectedStoppedAlloc.DesiredDescription = desiredDesc - expectedStoppedAlloc.DesiredStatus = AllocDesiredStatusStop - expectedStoppedAlloc.ClientStatus = AllocClientStatusLost - expectedStoppedAlloc.Job = nil - assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) - actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] - expectedPreemptedAlloc := &Allocation{ - ID: preemptedAlloc.ID, - PreemptedByAllocation: preemptingAllocID, - JobID: preemptedAlloc.JobID, - Namespace: preemptedAlloc.Namespace, - DesiredStatus: AllocDesiredStatusEvict, - DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), - AllocatedResources: preemptedAlloc.AllocatedResources, - TaskResources: preemptedAlloc.TaskResources, - SharedResources: preemptedAlloc.SharedResources, - } - assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) -} - func TestPlan_AppendStoppedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { t.Parallel() plan := &Plan{ @@ -2958,17 +2918,6 @@ func TestPlan_AppendPreemptedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { assert.Equal(t, expectedAlloc, appendedAlloc) } -func TestPlan_MsgPackTags(t *testing.T) { - t.Parallel() - planType := reflect.TypeOf(Plan{}) - - msgPackTags, _ := planType.FieldByName("_struct") - normalizeTag, _ := planType.FieldByName("NormalizeAllocs") - - assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) - assert.Equal(t, normalizeTag.Tag, reflect.StructTag(`codec:"-"`)) -} - func TestAllocation_MsgPackTags(t *testing.T) { t.Parallel() planType := reflect.TypeOf(Allocation{}) diff --git a/nomad/worker.go b/nomad/worker.go index 6cd8e0107fc..a50c58e68dd 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -284,8 +284,7 @@ func (w *Worker) invokeScheduler(eval *structs.Evaluation, token string) error { if eval.Type == structs.JobTypeCore { sched = NewCoreScheduler(w.srv, snap) } else { - allowPlanOptimization := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanDenormalization, true) - sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w, allowPlanOptimization) + sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w) if err != nil { return fmt.Errorf("failed to instantiate scheduler: %v", err) } @@ -312,7 +311,10 @@ func (w *Worker) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, scheduler. plan.EvalToken = w.evalToken // Normalize stopped and preempted allocs before RPC - plan.NormalizeAllocations() + normalizePlan := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanNormalization, true) + if normalizePlan { + plan.NormalizeAllocations() + } // Setup the request req := structs.PlanRequest{ diff --git a/nomad/worker_test.go b/nomad/worker_test.go index b2dc94f22b8..a03e739bfc3 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -19,11 +19,10 @@ import ( ) type NoopScheduler struct { - state scheduler.State - planner scheduler.Planner - eval *structs.Evaluation - allowPlanOptimization bool - err error + state scheduler.State + planner scheduler.Planner + eval *structs.Evaluation + err error } func (n *NoopScheduler) Process(eval *structs.Evaluation) error { @@ -38,11 +37,10 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { } func init() { - scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { + scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { n := &NoopScheduler{ - state: s, - planner: p, - allowPlanOptimization: allowPlanOptimization, + state: s, + planner: p, } return n } @@ -398,6 +396,7 @@ func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { s1 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.EnabledSchedulers = []string{structs.JobTypeService} + c.Build = "0.9.1" }) defer s1.Shutdown() testutil.WaitForLeader(t, s1.RPC) @@ -422,7 +421,6 @@ func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { EvalID: eval1.ID, NodeUpdate: make(map[string][]*structs.Allocation), NodePreemptions: make(map[string][]*structs.Allocation), - NormalizeAllocs: true, } desiredDescription := "desired desc" plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost) diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index ebdac6721cf..bcd8dc81ac5 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -5,8 +5,8 @@ import ( "time" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" ) @@ -77,10 +77,6 @@ type GenericScheduler struct { planner Planner batch bool - // Temporary flag introduced till the code for sending/committing full allocs in the Plan can - // be safely removed - allowPlanOptimization bool - eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -98,25 +94,23 @@ type GenericScheduler struct { } // NewServiceScheduler is a factory function to instantiate a new service scheduler -func NewServiceScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewServiceScheduler(logger log.Logger, state State, planner Planner) Scheduler { s := &GenericScheduler{ - logger: logger.Named("service_sched"), - state: state, - planner: planner, - batch: false, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("service_sched"), + state: state, + planner: planner, + batch: false, } return s } // NewBatchScheduler is a factory function to instantiate a new batch scheduler -func NewBatchScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewBatchScheduler(logger log.Logger, state State, planner Planner) Scheduler { s := &GenericScheduler{ - logger: logger.Named("batch_sched"), - state: state, - planner: planner, - batch: true, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("batch_sched"), + state: state, + planner: planner, + batch: true, } return s } @@ -229,7 +223,7 @@ func (s *GenericScheduler) process() (bool, error) { s.followUpEvals = nil // Create a plan - s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) + s.plan = s.eval.MakePlan(s.job) if !s.batch { // Get any existing deployment diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 81cfbd2ec6b..3613b4df8b3 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -1,336 +1,661 @@ package scheduler import ( - "fmt" - "reflect" + "fmt" + "reflect" "sort" - "testing" - "time" - - memdb "github.com/hashicorp/go-memdb" - "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/mock" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "testing" + "time" + + memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func IsPlanOptimizedStr(allowPlanOptimization bool) string { - return fmt.Sprintf("Is plan optimized: %v", allowPlanOptimization) -} +func TestServiceSched_JobRegister(t *testing.T) { + h := NewHarness(t) -func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a job that uses distinct host and has count 1 higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the eval has no spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different ports were used. + used := make(map[int]map[string]struct{}) + for _, alloc := range out { + for _, resource := range alloc.TaskResources { + for _, port := range resource.Networks[0].DynamicPorts { + nodeMap, ok := used[port.Value] + if !ok { + nodeMap = make(map[string]struct{}) + used[port.Value] = nodeMap + } + if _, ok := nodeMap[alloc.NodeID]; ok { + t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) + } + nodeMap[alloc.NodeID] = struct{}{} } + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } +func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { + h := NewHarness(t) - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Create a job + job := mock.Job() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure the plan allocated + plan := h.Plans[0] + planned := make(map[string]*structs.Allocation) + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + planned[alloc.ID] = alloc + } + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Update the job to force a rolling upgrade + updated := job.Copy() + updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 + noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State) + if err := h1.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure we have created only one new allocation + // Ensure a single plan + if len(h1.Plans) != 1 { + t.Fatalf("bad: %#v", h1.Plans) + } + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 10 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocations were placed on the same node as the older + // ones + for _, new := range newPlanned { + if new.PreviousAllocation == "" { + t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) + } + + old, ok := planned[new.PreviousAllocation] + if !ok { + t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) + } + if new.NodeID != old.NodeID { + t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) + } } } -func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job with count 2 and disk as 60GB so that only one allocation + // can fit + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - rack := "rack2" - if i < 5 { - rack = "rack1" - } - node.Meta["rack"] = rack - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 8 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - RTarget: "2", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the eval has a blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan allocated only one allocation + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Ensure only one allocation was placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 4 { - t.Fatalf("bad: %#v", plan) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) +func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { + h := NewHarness(t) - // Ensure all allocations placed - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure each node was only used twice - used := make(map[string]uint64) - for _, alloc := range out { - if count, _ := used[alloc.NodeID]; count > 2 { - t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) - } - used[alloc.NodeID]++ - } + // Create a job that uses distinct host and has count 1 higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} + +func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + rack := "rack2" + if i < 5 { + rack = "rack1" + } + node.Meta["rack"] = rack + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 8 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 4 { + t.Fatalf("bad: %#v", plan) } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Ensure each node was only used twice + used := make(map[string]uint64) + for _, alloc := range out { + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) + } + used[alloc.NodeID]++ + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - for i := 0; i < 2; i++ { - node := mock.Node() - node.Meta["ssd"] = "true" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 2; i++ { + node := mock.Node() + node.Meta["ssd"] = "true" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property only on one task group. - job := mock.Job() - job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.ssd}", - }) + // Create a job that uses distinct property only on one task group. + job := mock.Job() + job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.ssd}", + }) - job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + job.TaskGroups[1].Name = "tg2" + job.TaskGroups[1].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure all allocations placed + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } - // Ensure all allocations placed - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - h.AssertEvalStatus(t, structs.EvalStatusComplete) +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { + h := NewHarness(t) + assert := assert.New(t) + + // Create a job that uses distinct property over the node-id + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${node.unique.id}", }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 6; i++ { + node := mock.Node() + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + } + + // Create some allocations + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") + + // Update the count + job2 := job.Copy() + job2.TaskGroups[0].Count = 6 + assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) } + assert.Len(planned, 6, "Planned Allocations") + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + assert.Nil(err, "AllocsByJob") + + // Ensure all allocations placed + assert.Len(out, 6, "Placed Allocations") + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { - for _, allowPlanOptimization := range []bool{false, true} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - assert := assert.New(t) +// Test job registration with spread configured +func TestServiceSched_Spread(t *testing.T) { + assert := assert.New(t) - // Create a job that uses distinct property over the node-id + start := uint8(100) + step := uint8(10) + + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%d%% in dc1", start) + t.Run(name, func(t *testing.T) { + h := NewHarness(t) + remaining := uint8(100 - start) + // Create a job that uses spread over data center job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${node.unique.id}", + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + SpreadTarget: []*structs.SpreadTarget{ + { + Value: "dc1", + Percent: start, + }, + { + Value: "dc2", + Percent: remaining, + }, + }, }) assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - - // Create some nodes + // Create some nodes, half in dc2 var nodes []*structs.Node - for i := 0; i < 6; i++ { + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } nodes = append(nodes, node) assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node } - // Create some allocations - var allocs []*structs.Allocation - for i := 0; i < 3; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") - - // Update the count - job2 := job.Copy() - job2.TaskGroups[0].Count = 6 - assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") - // Create a mock evaluation to register the job eval := &structs.Evaluation{ Namespace: structs.DefaultNamespace, @@ -357,3577 +682,2644 @@ func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) // Ensure the plan allocated var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c } - assert.Len(planned, 6, "Planned Allocations") - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - assert.Nil(err, "AllocsByJob") + assert.Len(planned, 10, "Planned Allocations") - // Ensure all allocations placed - assert.Len(out, 6, "Placed Allocations") + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 10 - i + if i > 0 { + expectedCounts["dc2"] = i + } + require.Equal(t, expectedCounts, dcAllocsMap) h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - } -} - -// Test job registration with spread configured -func TestServiceSched_Spread(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - assert := assert.New(t) - - start := uint8(100) - step := uint8(10) - - for i := 0; i < 10; i++ { - name := fmt.Sprintf("%d%% in dc1", start) - t.Run(name, func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - remaining := uint8(100 - start) - // Create a job that uses spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - SpreadTarget: []*structs.SpreadTarget{ - { - Value: "dc1", - Percent: start, - }, - { - Value: "dc2", - Percent: remaining, - }, - }, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c - } - assert.Len(planned, 10, "Planned Allocations") - - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 10 - i - if i > 0 { - expectedCounts["dc2"] = i - } - require.Equal(t, expectedCounts, dcAllocsMap) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - start = start - step - } - }) + start = start - step } } // Test job registration with even spread across dc func TestServiceSched_EvenSpread(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - assert := assert.New(t) + assert := assert.New(t) + + h := NewHarness(t) + // Create a job that uses even spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } - h := NewHarness(t, allowPlanOptimization) - // Create a job that uses even spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Expect even split allocs across datacenter + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 5 + expectedCounts["dc2"] = 5 - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + require.Equal(t, expectedCounts, dcAllocsMap) - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") +func TestServiceSched_JobRegister_Annotate(t *testing.T) { + h := NewHarness(t) - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c - } - assert.Len(planned, 10, "Planned Allocations") + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Expect even split allocs across datacenter - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 5 - expectedCounts["dc2"] = 5 + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - require.Equal(t, expectedCounts, dcAllocsMap) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) } -} -func TestServiceSched_JobRegister_Annotate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + expected := &structs.DesiredUpdates{Place: 10} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + } +} - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) +func TestServiceSched_JobRegister_CountZero(t *testing.T) { + h := NewHarness(t) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - expected := &structs.DesiredUpdates{Place: 10} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) - } - }) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_CountZero(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobRegister_AllocFail(t *testing.T) { + h := NewHarness(t) + + // Create NO nodes + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) } -} -func TestServiceSched_JobRegister_AllocFail(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Create NO nodes - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { + t.Fatalf("bad: %#v", metrics) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Check queued allocations + queued := outEval.QueuedAllocations["web"] + if queued != 10 { + t.Fatalf("expected queued: %v, actual: %v", 10, queued) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { + h := NewHarness(t) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a full node + node := mock.Node() + node.ReservedResources = &structs.NodeReservedResources{ + Cpu: structs.NodeReservedCpuResources{ + CpuShares: node.NodeResources.Cpu.CpuShares, + }, + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create an ineligible node + node2 := mock.Node() + node2.Attributes["kernel.name"] = "windows" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a jobs + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan has created a follow up eval. + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + created := h.CreateEvals[0] + if created.Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", created) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + classes := created.ClassEligibility + if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { + t.Fatalf("bad: %#v", classes) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + if created.EscapedComputedClass { + t.Fatalf("bad: %#v", created) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { - t.Fatalf("bad: %#v", metrics) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Check queued allocations - queued := outEval.QueuedAllocations["web"] - if queued != 10 { - t.Fatalf("expected queued: %v, actual: %v", 10, queued) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) } -} + outEval := h.Evals[0] -func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a full node - node := mock.Node() - node.ReservedResources = &structs.NodeReservedResources{ - Cpu: structs.NodeReservedCpuResources{ - CpuShares: node.NodeResources.Cpu.CpuShares, - }, - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Create an ineligible node - node2 := mock.Node() - node2.Attributes["kernel.name"] = "windows" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Create a jobs - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { + t.Fatalf("bad: %#v", metrics) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { + h := NewHarness(t) + + // Create one node + node := mock.Node() + node.NodeClass = "class_0" + noErr(t, node.ComputeClass()) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job that constrains on a node class + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].Constraints = append(job.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "class_0", + Operand: "=", + }, + ) + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "web2" + tg2.Constraints[1].RTarget = "class_1" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan has created a follow up eval. - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 2 { + t.Fatalf("bad: %#v", plan) + } - created := h.CreateEvals[0] - if created.Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", created) - } + // Ensure two allocations placed + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } - classes := created.ClassEligibility - if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { - t.Fatalf("bad: %#v", classes) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - if created.EscapedComputedClass { - t.Fatalf("bad: %#v", created) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan failed to alloc one tg + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + metrics, ok := outEval.FailedTGAllocs[tg2.Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Check the coalesced failures + if metrics.CoalescedFailures != tg2.Count-1 { + t.Fatalf("bad: %#v", metrics) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } +// This test just ensures the scheduler handles the eval type to avoid +// regressions. +func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { + h := NewHarness(t) + + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerMaxPlans, + JobID: job.ID, + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { - t.Fatalf("bad: %#v", metrics) - } + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_Plan_Partial_Progress(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job with a high resource ask so that all the allocations can't + // be placed on a single node. + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create one node - node := mock.Node() - node.NodeClass = "class_0" - noErr(t, node.ComputeClass()) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job that constrains on a node class - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].Constraints = append(job.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "class_0", - Operand: "=", - }, - ) - tg2 := job.TaskGroups[0].Copy() - tg2.Name = "web2" - tg2.Constraints[1].RTarget = "class_1" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 2 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure two allocations placed - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan failed to alloc one tg - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - metrics, ok := outEval.FailedTGAllocs[tg2.Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Check the coalesced failures - if metrics.CoalescedFailures != tg2.Count-1 { - t.Fatalf("bad: %#v", metrics) - } + // Ensure only one allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) } -} -// This test just ensures the scheduler handles the eval type to avoid -// regressions. -func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) +func TestServiceSched_EvaluateBlockedEval(t *testing.T) { + h := NewHarness(t) + + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerMaxPlans, - JobID: job.ID, - } + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure that the eval was reblocked + if len(h.ReblockEvals) != 1 { + t.Fatalf("bad: %#v", h.ReblockEvals) + } + if h.ReblockEvals[0].ID != eval.ID { + t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the eval status was not updated + if len(h.Evals) != 0 { + t.Fatalf("Existing eval should not have status set") } } -func TestServiceSched_Plan_Partial_Progress(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { + h := NewHarness(t) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job with a high resource ask so that all the allocations can't - // be placed on a single node. - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the eval has no spawned blocked eval + if len(h.Evals) != 1 { + t.Fatalf("bad: %#v", h.Evals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure only one allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } + // Ensure the eval was not reblocked + if len(h.ReblockEvals) != 0 { + t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Ensure queued allocations is zero + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued: %v, actual: %v", 0, queued) } } -func TestServiceSched_EvaluateBlockedEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobModify(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the eval was reblocked - if len(h.ReblockEvals) != 1 { - t.Fatalf("bad: %#v", h.ReblockEvals) - } - if h.ReblockEvals[0].ID != eval.ID { - t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the eval status was not updated - if len(h.Evals) != 0 { - t.Fatalf("Existing eval should not have status set") - } - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +// Have a single node and submit a job. Increment the count such that all fit +// on the node but the node doesn't have enough resources to fit the new count + +// 1. This tests that we properly discount the resources of existing allocs. +func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create one node + node := mock.Node() + node.NodeResources.Cpu.CpuShares = 1000 + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job and set the task group count to zero. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with one allocation + job := mock.Job() + job.TaskGroups[0].Tasks[0].Resources.CPU = 256 + job2 := job.Copy() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + var allocs []*structs.Allocation + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job to count 3 + job2.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan didn't evicted the alloc + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the eval has no spawned blocked eval - if len(h.Evals) != 1 { - t.Fatalf("bad: %#v", h.Evals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure the plan had no failures + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + if outEval == nil || len(outEval.FailedTGAllocs) != 0 { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the eval was not reblocked - if len(h.ReblockEvals) != 0 { - t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") - } +func TestServiceSched_JobModify_CountZero(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job to be count zero + job2 := mock.Job() + job2.ID = job.ID + job2.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure queued allocations is zero - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued: %v, actual: %v", 0, queued) - } - }) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } -} -func TestServiceSched_JobModify(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure the plan didn't allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Update the job - job2 := mock.Job() - job2.ID = job.ID + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) +func TestServiceSched_JobModify_Rolling(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != desiredUpdates { + t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") } -} -// Have a single node and submit a job. Increment the count such that all fit -// on the node but the node doesn't have enough resources to fit the new count + -// 1. This tests that we properly discount the resources of existing allocs. -func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create one node - node := mock.Node() - node.NodeResources.Cpu.CpuShares = 1000 - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with one allocation - job := mock.Job() - job.TaskGroups[0].Tasks[0].Resources.CPU = 256 - job2 := job.Copy() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job to count 3 - job2.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan didn't evicted the alloc - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan had no failures - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - if outEval == nil || len(outEval.FailedTGAllocs) != 0 { - t.Fatalf("bad: %#v", outEval) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_JobModify_CountZero(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job to be count zero - job2 := mock.Job() - job2.ID = job.ID - job2.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan didn't allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_JobModify_Rolling(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != desiredUpdates { - t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } - }) - } -} - -// This tests that the old allocation is stopped before placing. -// It is critical to test that the updated job attempts to place more -// allocations as this allows us to assert that destructive changes are done -// first. -func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a node and clear the reserved resources - node := mock.Node() - node.ReservedResources = nil - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a resource ask that is the same as the resources available on the - // node - cpu := node.NodeResources.Cpu.CpuShares - mem := node.NodeResources.Memory.MemoryMB - - request := &structs.Resources{ - CPU: int(cpu), - MemoryMB: int(mem), - } - allocated := &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: cpu, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: mem, - }, - }, - }, - } - - // Generate a fake job with one alloc that consumes the whole node - job := mock.Job() - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Tasks[0].Resources = request - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.AllocatedResources = allocated - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Update the job to place more versions of the task group, drop the count - // and force destructive updates - job2 := job.Copy() - job2.TaskGroups[0].Count = 5 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 1 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 5 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } - }) - } -} - -func TestServiceSched_JobModify_Canaries(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 2 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - Canary: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted nothing - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - for _, canary := range planned { - if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { - t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { - t.Fatalf("bad: %#v", state) - } - - // Assert the canaries were added to the placed list - if len(state.PlacedCanaries) != desiredUpdates { - t.Fatalf("bad: %#v", state) - } - }) - } -} - -func TestServiceSched_JobModify_InPlace(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and create an older deployment - job := mock.Job() - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create allocs that are part of the old deployment - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = d.ID - alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) - } - } - } - - // Verify the deployment id was changed and health cleared - for _, alloc := range out { - if alloc.DeploymentID == d.ID { - t.Fatalf("bad: deployment id not cleared") - } else if alloc.DeploymentStatus != nil { - t.Fatalf("bad: deployment status not cleared") - } - } - }) - } -} - -func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - node.Meta["rack"] = fmt.Sprintf("rack%d", i) - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - oldJob := job.Copy() - oldJob.JobModifyIndex -= 1 - oldJob.TaskGroups[0].Count = 4 - - // Place 4 of 10 - var allocs []*structs.Allocation - for i := 0; i < 4; i++ { - alloc := mock.Alloc() - alloc.Job = oldJob - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", planned) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_JobDeregister_Purged(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Generate a fake job with allocations - job := mock.Job() - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all nodes - if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - if alloc.Job == nil { - t.Fatalf("bad: %#v", alloc) - } - } - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_JobDeregister_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) - - // Generate a fake job with allocations - job := mock.Job() - job.Stop = true - require.NoError(h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a summary where the queued allocs are set as we want to assert - // they get zeroed out. - summary := mock.JobSummary(job.ID) - web := summary.Summary["web"] - web.Queued = 2 - require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - require.NoError(h.Process(NewServiceScheduler, eval)) - - // Ensure a single plan - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan evicted all nodes - require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(err) - - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - require.NotNil(alloc.Job) - } - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - require.Empty(out) - - // Assert the job summary is cleared out - sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) - require.NoError(err) - require.NotNil(sout) - require.Contains(sout.Summary, "web") - webOut := sout.Summary["web"] - require.Zero(webOut.Queued) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_NodeDown(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - - // Cover each terminal case and ensure it doesn't change to lost - allocs[7].DesiredStatus = structs.AllocDesiredStatusRun - allocs[7].ClientStatus = structs.AllocClientStatusLost - allocs[8].DesiredStatus = structs.AllocDesiredStatusRun - allocs[8].ClientStatus = structs.AllocClientStatusFailed - allocs[9].DesiredStatus = structs.AllocDesiredStatusRun - allocs[9].ClientStatus = structs.AllocClientStatusComplete - - // Mark some allocs as running - for i := 0; i < 4; i++ { - out := allocs[i] - out.ClientStatus = structs.AllocClientStatusRunning - } - - // Mark appropriate allocs for migration - for i := 0; i < 7; i++ { - out := allocs[i] - out.DesiredTransition.Migrate = helper.BoolToPtr(true) - } - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Test the scheduler marked all non-terminal allocations as lost - if len(plan.NodeUpdate[node.ID]) != 7 { - t.Fatalf("bad: %#v", plan) - } - - for _, out := range plan.NodeUpdate[node.ID] { - if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad alloc: %#v", out) - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_NodeUpdate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Mark some allocs as running - ws := memdb.NewWatchSet() - for i := 0; i < 4; i++ { - out, _ := h.State.AllocByID(ws, allocs[i].ID) - out.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) - } - - // Create a mock evaluation which won't trigger any new placements - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_NodeDrain(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_NodeDrain_Down(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Set the desired state of the allocs to stop - var stop []*structs.Allocation - for i := 0; i < 6; i++ { - newAlloc := allocs[i].Copy() - newAlloc.ClientStatus = structs.AllocDesiredStatusStop - newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - stop = append(stop, newAlloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) - - // Mark some of the allocations as running - var running []*structs.Allocation - for i := 4; i < 6; i++ { - newAlloc := stop[i].Copy() - newAlloc.ClientStatus = structs.AllocClientStatusRunning - running = append(running, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) - - // Mark some of the allocations as complete - var complete []*structs.Allocation - for i := 6; i < 10; i++ { - newAlloc := allocs[i].Copy() - newAlloc.TaskStates = make(map[string]*structs.TaskState) - newAlloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - newAlloc.ClientStatus = structs.AllocClientStatusComplete - complete = append(complete, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 6 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure that all the allocations which were in running or pending state - // has been marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - sort.Strings(lostAllocs) - - var expectedLostAllocs []string - for i := 0; i < 6; i++ { - expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) - } - sort.Strings(expectedLostAllocs) - - if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { - t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} - -func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } - }) + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) } -} - -func TestServiceSched_RetryLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - h.Planner = &RejectPlan{h} - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) - }) + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) } } -func TestServiceSched_Reschedule_OnceNow(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } +// This tests that the old allocation is stopped before placing. +// It is critical to test that the updated job attempts to place more +// allocations as this allows us to assert that destructive changes are done +// first. +func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { + h := NewHarness(t) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: 5 * time.Second, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() + // Create a node and clear the reserved resources + node := mock.Node() + node.ReservedResources = nil + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a resource ask that is the same as the resources available on the + // node + cpu := node.NodeResources.Cpu.CpuShares + mem := node.NodeResources.Memory.MemoryMB - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID + request := &structs.Resources{ + CPU: int(cpu), + MemoryMB: int(mem), + } + allocated := &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: cpu, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: mem, + }, + }, + }, + } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with one alloc that consumes the whole node + job := mock.Job() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Resources = request + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + alloc := mock.Alloc() + alloc.AllocatedResources = allocated + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Update the job to place more versions of the task group, drop the count + // and force destructive updates + job2 := job.Copy() + job2.TaskGroups[0].Count = 5 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 1 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) + } - // Mark this alloc as failed again, should not get rescheduled - newAlloc.ClientStatus = structs.AllocClientStatusFailed + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 5 { + t.Fatalf("bad: %#v", plan) + } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Process the evaluation - err = h.Process(NewServiceScheduler, eval) - assert.Nil(err) - // Verify no new allocs were created this time - out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(3, len(out)) - }) + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) } } -// Tests that alloc reschedulable at a future time creates a follow up eval -func TestServiceSched_Reschedule_Later(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - delayDuration := 15 * time.Second - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: delayDuration, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() +func TestServiceSched_JobModify_Canaries(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 2 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + Canary: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now}} - failedAllocID := allocs[1].ID + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan evicted nothing + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Verify no new allocs were created - require.Equal(2, len(out)) + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Verify follow up eval was created for the failed alloc - alloc, err := h.State.AllocByID(ws, failedAllocID) - require.Nil(err) - require.NotEmpty(alloc.FollowupEvalID) + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { + t.Fatalf("bad: %#v", state) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { - t.Fatalf("bad: %#v", h.CreateEvals) - } - followupEval := h.CreateEvals[0] - require.Equal(now.Add(delayDuration), followupEval.WaitUntil) - }) + // Assert the canaries were added to the placed list + if len(state.PlacedCanaries) != desiredUpdates { + t.Fatalf("bad: %#v", state) } } -func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - maxRestartAttempts := 3 - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: maxRestartAttempts, - Interval: 30 * time.Minute, - Delay: 5 * time.Second, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) +func TestServiceSched_JobModify_InPlace(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.ClientStatus = structs.AllocClientStatusRunning - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} + // Generate a fake job with allocations and create an older deployment + job := mock.Job() + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create allocs that are part of the old deployment + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - expectedNumAllocs := 3 - expectedNumReschedTrackers := 1 + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - failedAllocId := allocs[1].ID - failedNodeID := allocs[1].NodeID + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - assert := assert.New(t) - for i := 0; i < maxRestartAttempts; i++ { - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that a new allocation got created with its restart tracker info - assert.Equal(expectedNumAllocs, len(out)) - - // Find the new alloc with ClientStatusPending - var pendingAllocs []*structs.Allocation - var prevFailedAlloc *structs.Allocation - - for _, alloc := range out { - if alloc.ClientStatus == structs.AllocClientStatusPending { - pendingAllocs = append(pendingAllocs, alloc) - } - if alloc.ID == failedAllocId { - prevFailedAlloc = alloc - } - } - assert.Equal(1, len(pendingAllocs)) - newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) - - // Verify the previous NodeID in the most recent reschedule event - reschedEvents := newAlloc.RescheduleTracker.Events - assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) - assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) - - // Verify that the next alloc of the failed alloc is the newly rescheduled alloc - assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) - - // Mark this alloc as failed again - newAlloc.ClientStatus = structs.AllocClientStatusFailed - newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-12 * time.Second), - FinishedAt: now.Add(-10 * time.Second)}} - - failedAllocId = newAlloc.ID - failedNodeID = newAlloc.NodeID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expectedNumAllocs += 1 - expectedNumReschedTrackers += 1 + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) } + } + } - // Process last eval again, should not reschedule - err := h.Process(NewServiceScheduler, eval) - assert.Nil(err) - - // Verify no new allocs were created because restart attempts were exhausted - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts - }) + // Verify the deployment id was changed and health cleared + for _, alloc := range out { + if alloc.DeploymentID == d.ID { + t.Fatalf("bad: deployment id not cleared") + } else if alloc.DeploymentStatus != nil { + t.Fatalf("bad: deployment status not cleared") + } } } -// Tests that old reschedule attempts are pruned -func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - DelayFunction: "exponential", - MaxDelay: 1 * time.Hour, - Delay: 5 * time.Second, - Unlimited: true, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) +func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + node.Meta["rack"] = fmt.Sprintf("rack%d", i) + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - now := time.Now() - // Mark allocations as failed with restart info - allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-15 * time.Minute)}} - allocs[1].ClientStatus = structs.AllocClientStatusFailed + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + oldJob := job.Copy() + oldJob.JobModifyIndex -= 1 + oldJob.TaskGroups[0].Count = 4 + + // Place 4 of 10 + var allocs []*structs.Allocation + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = oldJob + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - allocs[1].RescheduleTracker = &structs.RescheduleTracker{ - Events: []*structs.RescheduleEvent{ - {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), - PrevAllocID: uuid.Generate(), - PrevNodeID: uuid.Generate(), - Delay: 5 * time.Second, - }, - {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 10 * time.Second, - }, - {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 20 * time.Second, - }, - {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 40 * time.Second, - }, - {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 80 * time.Second, - }, - {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 160 * time.Second, - }, - }, - } - expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] - expectedDelay := 320 * time.Second - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", planned) + } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - // Verify that the new alloc copied the last 5 reschedule attempts - assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] - // Verify that the failed alloc ID is in the most recent reschedule event - assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) - // Verify that the delay value was captured correctly - assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) - }) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } -} -// Tests that deployments with failed allocs result in placements as long as the -// deployment is running. -func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - for _, failedDeployment := range []bool{false, true} { - t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and a reschedule policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) - - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - if failedDeployment { - deployment.Status = structs.DeploymentStatusFailed - } - - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed in the past - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", - StartedAt: time.Now().Add(-12 * time.Hour), - FinishedAt: time.Now().Add(-10 * time.Hour)}} - allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) - - if failedDeployment { - // Verify no plan created - require.Len(h.Plans, 0) - } else { - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - } - }) - } - }) + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestBatchSched_Run_CompleteAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobDeregister_Purged(t *testing.T) { + h := NewHarness(t) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Generate a fake job with allocations + job := mock.Job() - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure no plan as it should be a no-op - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan evicted all nodes + if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + if alloc.Job == nil { + t.Fatalf("bad: %#v", alloc) + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestBatchSched_Run_FailedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobDeregister_Stopped(t *testing.T) { + h := NewHarness(t) + require := require.New(t) + + // Generate a fake job with allocations + job := mock.Job() + job.Stop = true + require.NoError(h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a summary where the queued allocs are set as we want to assert + // they get zeroed out. + summary := mock.JobSummary(job.ID) + web := summary.Summary["web"] + web.Queued = 2 + require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Process the evaluation + require.NoError(h.Process(NewServiceScheduler, eval)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - tgName := job.TaskGroups[0].Name - now := time.Now() - - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Ensure a single plan + require.Len(h.Plans, 1) + plan := h.Plans[0] - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan evicted all nodes + require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(err) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + require.NotNil(alloc.Job) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + require.Empty(out) - // Ensure a replacement alloc was placed. - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Assert the job summary is cleared out + sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) + require.NoError(err) + require.NotNil(sout) + require.Contains(sout.Summary, "web") + webOut := sout.Summary["web"] + require.Zero(webOut.Queued) - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) +func TestServiceSched_NodeDown(t *testing.T) { + h := NewHarness(t) + + // Register a node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) } -} -func TestBatchSched_Run_LostAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Cover each terminal case and ensure it doesn't change to lost + allocs[7].DesiredStatus = structs.AllocDesiredStatusRun + allocs[7].ClientStatus = structs.AllocClientStatusLost + allocs[8].DesiredStatus = structs.AllocDesiredStatusRun + allocs[8].ClientStatus = structs.AllocClientStatusFailed + allocs[9].DesiredStatus = structs.AllocDesiredStatusRun + allocs[9].ClientStatus = structs.AllocClientStatusComplete + + // Mark some allocs as running + for i := 0; i < 4; i++ { + out := allocs[i] + out.ClientStatus = structs.AllocClientStatusRunning + } - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Mark appropriate allocs for migration + for i := 0; i < 7; i++ { + out := allocs[i] + out.DesiredTransition.Migrate = helper.BoolToPtr(true) + } - // Create a job - job := mock.Job() - job.ID = "my-job" - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Desired = 3 - // Mark one as lost and then schedule - // [(0, run, running), (1, run, running), (1, stop, lost)] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create two running allocations - var allocs []*structs.Allocation - for i := 0; i <= 1; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[1]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Test the scheduler marked all non-terminal allocations as lost + if len(plan.NodeUpdate[node.ID]) != 7 { + t.Fatalf("bad: %#v", plan) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, out := range plan.NodeUpdate[node.ID] { + if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad alloc: %#v", out) + } + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } +func TestServiceSched_NodeUpdate(t *testing.T) { + h := NewHarness(t) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Mark some allocs as running + ws := memdb.NewWatchSet() + for i := 0; i < 4; i++ { + out, _ := h.State.AllocByID(ws, allocs[i].ID) + out.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation which won't trigger any new placements + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a replacement alloc was placed. - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + } - // Assert that we have the correct number of each alloc name - expected := map[string]int{ - "my-job.web[0]": 1, - "my-job.web[1]": 2, - "my-job.web[2]": 1, - } - actual := make(map[string]int, 3) - for _, alloc := range out { - actual[alloc.Name] += 1 - } - require.Equal(t, actual, expected) + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) +func TestServiceSched_NodeDrain(t *testing.T) { + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } -} -func TestServiceSched_JobRegister(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the eval has no spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } +func TestServiceSched_NodeDrain_Down(t *testing.T) { + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Set the desired state of the allocs to stop + var stop []*structs.Allocation + for i := 0; i < 6; i++ { + newAlloc := allocs[i].Copy() + newAlloc.ClientStatus = structs.AllocDesiredStatusStop + newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + stop = append(stop, newAlloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) + + // Mark some of the allocations as running + var running []*structs.Allocation + for i := 4; i < 6; i++ { + newAlloc := stop[i].Copy() + newAlloc.ClientStatus = structs.AllocClientStatusRunning + running = append(running, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) + + // Mark some of the allocations as complete + var complete []*structs.Allocation + for i := 6; i < 10; i++ { + newAlloc := allocs[i].Copy() + newAlloc.TaskStates = make(map[string]*structs.TaskState) + newAlloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + newAlloc.ClientStatus = structs.AllocClientStatusComplete + complete = append(complete, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure different ports were used. - used := make(map[int]map[string]struct{}) - for _, alloc := range out { - for _, resource := range alloc.TaskResources { - for _, port := range resource.Networks[0].DynamicPorts { - nodeMap, ok := used[port.Value] - if !ok { - nodeMap = make(map[string]struct{}) - used[port.Value] = nodeMap - } - if _, ok := nodeMap[alloc.NodeID]; ok { - t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) - } - nodeMap[alloc.NodeID] = struct{}{} - } - } - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 6 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure that all the allocations which were in running or pending state + // has been marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + sort.Strings(lostAllocs) + + var expectedLostAllocs []string + for i := 0; i < 6; i++ { + expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) + } + sort.Strings(expectedLostAllocs) + + if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { + t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a job - job := mock.Job() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } +} - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestServiceSched_RetryLimit(t *testing.T) { + h := NewHarness(t) + h.Planner = &RejectPlan{h} - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan allocated - plan := h.Plans[0] - planned := make(map[string]*structs.Allocation) - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - planned[alloc.ID] = alloc - } - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Update the job to force a rolling upgrade - updated := job.Copy() - updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 - noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - if err := h1.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure we have created only one new allocation - // Ensure a single plan - if len(h1.Plans) != 1 { - t.Fatalf("bad: %#v", h1.Plans) - } - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 10 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocations were placed on the same node as the older - // ones - for _, new := range newPlanned { - if new.PreviousAllocation == "" { - t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - old, ok := planned[new.PreviousAllocation] - if !ok { - t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) - } - if new.NodeID != old.NodeID { - t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) - } - } - }) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) } -func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_Reschedule_OnceNow(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: 5 * time.Second, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job with count 2 and disk as 60GB so that only one allocation - // can fit - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - ID: uuid.Generate(), - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) + + // Mark this alloc as failed again, should not get rescheduled + newAlloc.ClientStatus = structs.AllocClientStatusFailed + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err = h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created this time + out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(3, len(out)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] +} - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } +// Tests that alloc reschedulable at a future time creates a follow up eval +func TestServiceSched_Reschedule_Later(t *testing.T) { + h := NewHarness(t) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the eval has a blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + delayDuration := 15 * time.Second + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: delayDuration, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now}} + failedAllocID := allocs[1].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated only one allocation - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure only one allocation was placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Verify no new allocs were created + require.Equal(2, len(out)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Verify follow up eval was created for the failed alloc + alloc, err := h.State.AllocByID(ws, failedAllocID) + require.Nil(err) + require.NotEmpty(alloc.FollowupEvalID) + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { + t.Fatalf("bad: %#v", h.CreateEvals) } + followupEval := h.CreateEvals[0] + require.Equal(now.Add(delayDuration), followupEval.WaitUntil) } -func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) +func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - tgName := job.TaskGroups[0].Name - now := time.Now() - - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + maxRestartAttempts := 3 + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: maxRestartAttempts, + Interval: 30 * time.Minute, + Delay: 5 * time.Second, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + expectedNumAllocs := 3 + expectedNumReschedTrackers := 1 + + failedAllocId := allocs[1].ID + failedNodeID := allocs[1].NodeID + + assert := assert.New(t) + for i := 0; i < maxRestartAttempts; i++ { + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + noErr(t, err) + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that a new allocation got created with its restart tracker info + assert.Equal(expectedNumAllocs, len(out)) + + // Find the new alloc with ClientStatusPending + var pendingAllocs []*structs.Allocation + var prevFailedAlloc *structs.Allocation + + for _, alloc := range out { + if alloc.ClientStatus == structs.AllocClientStatusPending { + pendingAllocs = append(pendingAllocs, alloc) + } + if alloc.ID == failedAllocId { + prevFailedAlloc = alloc + } + } + assert.Equal(1, len(pendingAllocs)) + newAlloc := pendingAllocs[0] + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) + + // Verify the previous NodeID in the most recent reschedule event + reschedEvents := newAlloc.RescheduleTracker.Events + assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) + assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) + + // Verify that the next alloc of the failed alloc is the newly rescheduled alloc + assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) + + // Mark this alloc as failed again + newAlloc.ClientStatus = structs.AllocClientStatusFailed + newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-12 * time.Second), + FinishedAt: now.Add(-10 * time.Second)}} + + failedAllocId = newAlloc.ID + failedNodeID = newAlloc.NodeID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + expectedNumAllocs += 1 + expectedNumReschedTrackers += 1 + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process last eval again, should not reschedule + err := h.Process(NewServiceScheduler, eval) + assert.Nil(err) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Verify no new allocs were created because restart attempts were exhausted + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts +} - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 1 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } - }) +// Tests that old reschedule attempts are pruned +func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } -} -func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a successful alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + DelayFunction: "exponential", + MaxDelay: 1 * time.Hour, + Delay: 5 * time.Second, + Unlimited: true, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + now := time.Now() + // Mark allocations as failed with restart info + allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-15 * time.Minute)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed + + allocs[1].RescheduleTracker = &structs.RescheduleTracker{ + Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + Delay: 5 * time.Second, + }, + {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 10 * time.Second, + }, + {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 20 * time.Second, + }, + {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 40 * time.Second, + }, + {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 80 * time.Second, + }, + {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 160 * time.Second, + }, + }, + } + expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] + expectedDelay := 320 * time.Second + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to rerun the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + // Verify that the new alloc copied the last 5 reschedule attempts + assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) - // Ensure no replacement alloc was placed. - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] + // Verify that the failed alloc ID is in the most recent reschedule event + assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) + // Verify that the delay value was captured correctly + assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } } -// This test checks that terminal allocations that receive an in-place updated -// are not added to the plan -func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t) + require := require.New(t) // Create some nodes var nodes []*structs.Node for i := 0; i < 10; i++ { @@ -3936,241 +3328,519 @@ func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Generate a fake job with allocations + // Generate a fake job with allocations and a reschedule policy. job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + } + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) + + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed + } + + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) var allocs []*structs.Allocation - for i := 0; i < 10; i++ { + for i := 0; i < 2; i++ { alloc := mock.Alloc() alloc.Job = job alloc.JobID = job.ID alloc.NodeID = nodes[i].ID alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.DeploymentID = deployment.ID allocs = append(allocs, alloc) } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - // Create a mock evaluation to trigger the job + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation eval := &structs.Evaluation{ Namespace: structs.DefaultNamespace, ID: uuid.Generate(), Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, + TriggeredBy: structs.EvalTriggerNodeUpdate, JobID: job.ID, Status: structs.EvalStatusPending, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans[0]) + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } } }) } } -// This test ensures that terminal jobs from older versions are ignored. -func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestBatchSched_Run_CompleteAlloc(t *testing.T) { + h := NewHarness(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - job2.Type = structs.JobTypeBatch - job2.Version++ - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - allocs = nil - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job2 - alloc.JobID = job2.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - }, - } - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Ensure no plan as it should be a no-op + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure no allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Ensure a plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - }) + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} + +func TestBatchSched_Run_FailedAlloc(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } + + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// This test asserts that an allocation from an old job that is running on a -// drained node is cleaned up. -func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a running alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create an update job - job2 := job.Copy() - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - job2.Version++ - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) +func TestBatchSched_Run_LostAlloc(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.Job() + job.ID = "my-job" + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Desired = 3 + // Mark one as lost and then schedule + // [(0, run, running), (1, run, running), (1, stop, lost)] + + // Create two running allocations + var allocs []*structs.Allocation + for i := 0; i <= 1; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[1]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Assert that we have the correct number of each alloc name + expected := map[string]int{ + "my-job.web[0]": 1, + "my-job.web[1]": 2, + "my-job.web[2]": 1, + } + actual := make(map[string]int, 3) + for _, alloc := range out { + actual[alloc.Name] += 1 + } + require.Equal(t, actual, expected) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} + +func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { + h := NewHarness(t) + + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 1 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } +} + +func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { + h := NewHarness(t) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a successful alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to rerun the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no replacement alloc was placed. + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - plan := h.Plans[0] + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the plan evicted 1 - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } +// This test checks that terminal allocations that receive an in-place updated +// are not added to the plan +func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan places 1 - if len(plan.NodeAllocation[node2.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to trigger the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans[0]) } } -// This test asserts that an allocation from a job that is complete on a -// drained node is ignored up. -func TestBatchSched_NodeDrain_Complete(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = make(map[string]*structs.TaskState) - alloc.TaskStates["web"] = &structs.TaskState{ +// This test ensures that terminal jobs from older versions are ignored. +func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + job2.Type = structs.JobTypeBatch + job2.Version++ + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + allocs = nil + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job2 + alloc.JobID = job2.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { State: structs.TaskStateDead, Events: []*structs.TaskEvent{ { @@ -4178,416 +3848,531 @@ func TestBatchSched_NodeDrain_Complete(t *testing.T) { ExitCode: 0, }, }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + }, + } + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure a plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } +} - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } +// This test asserts that an allocation from an old job that is running on a +// drained node is cleaned up. +func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { + h := NewHarness(t) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a running alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create an update job + job2 := job.Copy() + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + job2.Version++ + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + plan := h.Plans[0] + + // Ensure the plan evicted 1 + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan places 1 + if len(plan.NodeAllocation[node2.ID]) != 1 { + t.Fatalf("bad: %#v", plan) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// This is a slightly odd test but it ensures that we handle a scale down of a -// task group's count and that it works even if all the allocs have the same -// name. -func TestBatchSched_ScaleDown_SameName(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +// This test asserts that an allocation from a job that is complete on a +// drained node is ignored up. +func TestBatchSched_NodeDrain_Complete(t *testing.T) { + h := NewHarness(t) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = make(map[string]*structs.TaskState) + alloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a few running alloc - var allocs []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +// This is a slightly odd test but it ensures that we handle a scale down of a +// task group's count and that it works even if all the allocs have the same +// name. +func TestBatchSched_ScaleDown_SameName(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a few running alloc + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - plan := h.Plans[0] + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan evicted 4 of the 5 - if len(plan.NodeUpdate[node.ID]) != 4 { - t.Fatalf("bad: %#v", plan) - } + plan := h.Plans[0] - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan evicted 4 of the 5 + if len(plan.NodeUpdate[node.ID]) != 4 { + t.Fatalf("bad: %#v", plan) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestGenericSched_ChainedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State) + job1 := mock.Job() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + job1.TaskGroups[0].Count = 12 + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - job1 := mock.Job() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - job1.TaskGroups[0].Count = 12 - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewServiceScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - if err := h1.Process(NewServiceScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + plan := h1.Plans[0] - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue - } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue } - sort.Strings(prevAllocs) + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) - } - }) + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) } } func TestServiceSched_NodeDrain_Sticky(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create an alloc on the draining node - alloc := mock.Alloc() - alloc.Name = "my-job.web[0]" - alloc.NodeID = node.ID - alloc.Job.TaskGroups[0].Count = 1 - alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: alloc.Job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + h := NewHarness(t) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create an alloc on the draining node + alloc := mock.Alloc() + alloc.Name = "my-job.web[0]" + alloc.NodeID = node.ID + alloc.Job.TaskGroups[0].Count = 1 + alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: alloc.Job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan didn't create any new allocations - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) } + + // Ensure the plan didn't create any new allocations + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is stopped, the scheduler properly cancels // an outstanding deployment. func TestServiceSched_CancelDeployment_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Generate a fake job - job := mock.Job() - job.JobModifyIndex = job.CreateIndex + 1 - job.ModifyIndex = job.CreateIndex + 1 - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment - d := mock.Deployment() - d.JobID = job.ID - d.JobCreateIndex = job.CreateIndex - d.JobModifyIndex = job.JobModifyIndex - 1 - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Generate a fake job + job := mock.Job() + job.JobModifyIndex = job.CreateIndex + 1 + job.ModifyIndex = job.CreateIndex + 1 + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a deployment + d := mock.Deployment() + d.JobID = job.ID + d.JobCreateIndex = job.CreateIndex + d.JobModifyIndex = job.JobModifyIndex - 1 + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) - } + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is updated and had an old deployment, the scheduler properly cancels // the deployment. func TestServiceSched_CancelDeployment_NewerJob(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Generate a fake job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment for an old version of the job - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Upsert again to bump job version - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to kick the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Generate a fake job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a deployment for an old version of the job + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Upsert again to bump job version + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to kick the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) - } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if out == nil { + t.Fatalf("No deployment for job") } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) + } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // Various table driven tests for carry forward // of past reschedule events func Test_updateRescheduleTracker(t *testing.T) { + t1 := time.Now().UTC() alloc := mock.Alloc() prevAlloc := mock.Alloc() @@ -4818,4 +4603,5 @@ func Test_updateRescheduleTracker(t *testing.T) { require.Equal(tc.expectedRescheduleEvents, alloc.RescheduleTracker.Events) }) } + } diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 76a0a69408c..639b2b8cf28 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -28,7 +28,7 @@ var BuiltinSchedulers = map[string]Factory{ // NewScheduler is used to instantiate and return a new scheduler // given the scheduler name, initial state, and planner. -func NewScheduler(name string, logger log.Logger, state State, planner Planner, allowPlanOptimization bool) (Scheduler, error) { +func NewScheduler(name string, logger log.Logger, state State, planner Planner) (Scheduler, error) { // Lookup the factory function factory, ok := BuiltinSchedulers[name] if !ok { @@ -36,12 +36,12 @@ func NewScheduler(name string, logger log.Logger, state State, planner Planner, } // Instantiate the scheduler - sched := factory(logger, state, planner, allowPlanOptimization) + sched := factory(logger, state, planner) return sched, nil } // Factory is used to instantiate a new Scheduler -type Factory func(log.Logger, State, Planner, bool) Scheduler +type Factory func(log.Logger, State, Planner) Scheduler // Scheduler is the top level instance for a scheduler. A scheduler is // meant to only encapsulate business logic, pushing the various plumbing diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index 59edb31abb5..dc0966341ac 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -4,7 +4,7 @@ import ( "fmt" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" ) @@ -23,10 +23,6 @@ type SystemScheduler struct { state State planner Planner - // Temporary flag introduced till the code for sending/committing full allocs in the Plan can - // be safely removed - allowPlanOptimization bool - eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -45,12 +41,11 @@ type SystemScheduler struct { // NewSystemScheduler is a factory function to instantiate a new system // scheduler. -func NewSystemScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewSystemScheduler(logger log.Logger, state State, planner Planner) Scheduler { return &SystemScheduler{ - logger: logger.Named("system_sched"), - state: state, - planner: planner, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("system_sched"), + state: state, + planner: planner, } } @@ -115,7 +110,7 @@ func (s *SystemScheduler) process() (bool, error) { } // Create a plan - s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) + s.plan = s.eval.MakePlan(s.job) // Reset the failed allocations s.failedTGAllocs = nil diff --git a/scheduler/system_sched_test.go b/scheduler/system_sched_test.go index 2509697de93..9461fd85eaa 100644 --- a/scheduler/system_sched_test.go +++ b/scheduler/system_sched_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hashicorp/go-memdb" + memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -15,1964 +15,1870 @@ import ( "github.com/stretchr/testify/require" ) -// COMPAT 0.11: Currently, all the tests run for 2 cases: -// 1) Allow plan optimization -// 2) Not allowing plan optimization -// The code for not allowing plan optimizations is in place to allow for safer cluster upgrades, -// and backwards compatibility with the existing raft logs. The older code will be removed later, -// and these tests should then no longer include the testing for case (2) - func TestSystemSched_JobRegister(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure no allocations are queued - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no allocations are queued + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan allocated - plan := h.Plans[0] - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Get an allocation and mark it as failed - alloc := planned[4].Copy() - alloc.ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - if err := h1.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan allocated + plan := h.Plans[0] + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure we have created only one new allocation - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 1 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocation was placed on the same node as the older - // one - if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { - t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) - } - }) + // Get an allocation and mark it as failed + alloc := planned[4].Copy() + alloc.ClientStatus = structs.AllocClientStatusFailed + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State) + if err := h1.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure we have created only one new allocation + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 1 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocation was placed on the same node as the older + // one + if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { + t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) } } func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create another job with a lot of disk resource ask so that it doesn't fit - // the node - job1 := mock.SystemJob() - job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create another job with a lot of disk resource ask so that it doesn't fit + // the node + job1 := mock.SystemJob() + job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Create a new harness to test the scheduling result for the second job - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a new harness to test the scheduling result for the second job + h1 := NewHarnessWithState(t, h.State) + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) - noErr(t, err) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - }) + out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) + noErr(t, err) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } } func TestSystemSched_ExhaustResources(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create a service job which consumes most of the system resources - svcJob := mock.Job() - svcJob.TaskGroups[0].Count = 1 - svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: svcJob.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: svcJob.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create a service job which consumes most of the system resources + svcJob := mock.Job() + svcJob.TaskGroups[0].Count = 1 + svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: svcJob.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: svcJob.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a system job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Create a system job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // System scheduler will preempt the service job and would have placed eval1 - require := require.New(t) + // System scheduler will preempt the service job and would have placed eval1 + require := require.New(t) - newPlan := h.Plans[1] - require.Len(newPlan.NodeAllocation, 1) - require.Len(newPlan.NodePreemptions, 1) + newPlan := h.Plans[1] + require.Len(newPlan.NodeAllocation, 1) + require.Len(newPlan.NodePreemptions, 1) - for _, allocList := range newPlan.NodeAllocation { - require.Len(allocList, 1) - require.Equal(job.ID, allocList[0].JobID) - } + for _, allocList := range newPlan.NodeAllocation { + require.Len(allocList, 1) + require.Equal(job.ID, allocList[0].JobID) + } - for _, allocList := range newPlan.NodePreemptions { - require.Len(allocList, 1) - alloc, err := h.State.AllocByID(nil, allocList[0].ID) - noErr(t, err) - require.Equal(svcJob.ID, alloc.JobID) - } - // Ensure that we have no queued allocations on the second eval - queued := h.Evals[1].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } - }) + for _, allocList := range newPlan.NodePreemptions { + require.Len(allocList, 1) + require.Equal(svcJob.ID, allocList[0].JobID) + } + // Ensure that we have no queued allocations on the second eval + queued := h.Evals[1].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) } } func TestSystemSched_JobRegister_Annotate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - if i < 9 { - node.NodeClass = "foo" - } else { - node.NodeClass = "bar" - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job constraining on node class - job := mock.SystemJob() - fooConstraint := &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "foo", - Operand: "==", - } - job.Constraints = append(job.Constraints, fooConstraint) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + if i < 9 { + node.NodeClass = "foo" + } else { + node.NodeClass = "bar" + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job constraining on node class + job := mock.SystemJob() + fooConstraint := &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "foo", + Operand: "==", + } + job.Constraints = append(job.Constraints, fooConstraint) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 9 { - t.Fatalf("bad: %#v %d", planned, len(planned)) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 9 { + t.Fatalf("bad: %#v %d", planned, len(planned)) + } - // Ensure all allocations placed - if len(out) != 9 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure all allocations placed + if len(out) != 9 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - expected := &structs.DesiredUpdates{Place: 9} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - } - }) + expected := &structs.DesiredUpdates{Place: 9} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) } } func TestSystemSched_JobRegister_AddNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a new node. - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a new node. + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan had no node updates - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Log(len(update)) - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated on the new node - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan had no node updates + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Log(len(update)) + t.Fatalf("bad: %#v", plan) + } - // Ensure it allocated on the right node - if _, ok := plan.NodeAllocation[node.ID]; !ok { - t.Fatalf("allocated on wrong node: %#v", plan) - } + // Ensure the plan allocated on the new node + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure it allocated on the right node + if _, ok := plan.NodeAllocation[node.ID]; !ok { + t.Fatalf("allocated on wrong node: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 11 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 11 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_AllocFail(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create NO nodes - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan as this should be a no-op. - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + h := NewHarness(t) + + // Create NO nodes + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no plan as this should be a no-op. + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = "my-job.web[0]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = "my-job.web[0]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify_Rolling(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - job2.Update = structs.UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 5, - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + job2.Update = structs.UpdateStrategy{ + Stagger: 30 * time.Second, + MaxParallel: 5, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a follow up eval was created - eval = h.Evals[0] - if eval.NextEval == "" { - t.Fatalf("missing next eval") - } + // Ensure a follow up eval was created + eval = h.Evals[0] + if eval.NextEval == "" { + t.Fatalf("missing next eval") + } - // Check for create - if len(h.CreateEvals) == 0 { - t.Fatalf("missing created eval") - } - create := h.CreateEvals[0] - if eval.NextEval != create.ID { - t.Fatalf("ID mismatch") - } - if create.PreviousEval != eval.ID { - t.Fatalf("missing previous eval") - } + // Check for create + if len(h.CreateEvals) == 0 { + t.Fatalf("missing created eval") + } + create := h.CreateEvals[0] + if eval.NextEval != create.ID { + t.Fatalf("ID mismatch") + } + if create.PreviousEval != eval.ID { + t.Fatalf("missing previous eval") + } - if create.TriggeredBy != structs.EvalTriggerRollingUpdate { - t.Fatalf("bad: %#v", create) - } - }) + if create.TriggeredBy != structs.EvalTriggerRollingUpdate { + t.Fatalf("bad: %#v", create) } } func TestSystemSched_JobModify_InPlace(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) - } - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) } - }) + } } } func TestSystemSched_JobDeregister_Purged(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobDeregister_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDown(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a down node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a down node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if p := planned[0]; p.DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && + p.ClientStatus != structs.AllocClientStatusLost { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain_Down(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the allocation is marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - expected := []string{alloc.ID} + // Ensure that the allocation is marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + expected := []string{alloc.ID} - if !reflect.DeepEqual(lostAllocs, expected) { - t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if !reflect.DeepEqual(lostAllocs, expected) { + t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) } + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Log(len(planned)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Log(len(planned)) + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeUpdate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_RetryLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - h.Planner = &RejectPlan{h} - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + h.Planner = &RejectPlan{h} - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) - }) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) } // This test ensures that the scheduler doesn't increment the queued allocation // count for a task group when allocations can't be created on currently // available nodes because of constrain mismatches. func TestSystemSched_Queued_With_Constraints(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - node.Attributes["kernel.name"] = "darwin" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a system job which can't be placed on the node - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register a node + node := mock.Node() + node.Attributes["kernel.name"] = "darwin" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a system job which can't be placed on the node + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } - }) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) } } func TestSystemSched_ChainedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - job1 := mock.SystemJob() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Insert two more nodes - for i := 0; i < 2; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State) + job1 := mock.SystemJob() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Insert two more nodes + for i := 0; i < 2; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue - } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + plan := h1.Plans[0] - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue } - }) + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) + + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } + + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) } } func TestSystemSched_PlanWithDrainedNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.Drain = true - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create an allocation on each node - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - alloc.TaskGroup = "web" - - alloc2 := mock.Alloc() - alloc2.Job = job - alloc2.JobID = job.ID - alloc2.NodeID = node2.ID - alloc2.Name = "my-job.web2[0]" - alloc2.TaskGroup = "web2" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.Drain = true + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create an allocation on each node + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + alloc.TaskGroup = "web" + + alloc2 := mock.Alloc() + alloc2.Job = job + alloc2.JobID = job.ID + alloc2.NodeID = node2.ID + alloc2.Name = "my-job.web2[0]" + alloc2.TaskGroup = "web2" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the alloc on the failed node - planned := plan.NodeUpdate[node.ID] - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan didn't place - if len(plan.NodeAllocation) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted the alloc on the failed node + planned := plan.NodeUpdate[node.ID] + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan didn't place + if len(plan.NodeAllocation) != 0 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - qa := h.Evals[0].QueuedAllocations - if qa["web"] != 0 || qa["web2"] != 0 { - t.Fatalf("bad queued allocations %#v", qa) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + qa := h.Evals[0].QueuedAllocations + if qa["web"] != 0 || qa["web2"] != 0 { + t.Fatalf("bad queued allocations %#v", qa) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_Preemption(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create nodes - var nodes []*structs.Node - for i := 0; i < 2; i++ { - node := mock.Node() - //TODO(preetha): remove in 0.11 - node.Resources = &structs.Resources{ - CPU: 3072, - MemoryMB: 5034, - DiskMB: 20 * 1024, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - node.NodeResources = &structs.NodeResources{ - Cpu: structs.NodeCpuResources{ - CpuShares: 3072, - }, - Memory: structs.NodeMemoryResources{ - MemoryMB: 5034, - }, - Disk: structs.NodeDiskResources{ - DiskMB: 20 * 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - nodes = append(nodes, node) - } - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, + h := NewHarness(t) + + // Create nodes + var nodes []*structs.Node + for i := 0; i < 2; i++ { + node := mock.Node() + //TODO(preetha): remove in 0.11 + node.Resources = &structs.Resources{ + CPU: 3072, + MemoryMB: 5034, + DiskMB: 20 * 1024, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, }, - }) - - // Create some low priority batch jobs and allocations for them - // One job uses a reserved port - job1 := mock.BatchJob() - job1.Type = structs.JobTypeBatch - job1.Priority = 20 - job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ + }, + } + node.NodeResources = &structs.NodeResources{ + Cpu: structs.NodeCpuResources{ + CpuShares: 3072, + }, + Memory: structs.NodeMemoryResources{ + MemoryMB: 5034, + }, + Disk: structs.NodeDiskResources{ + DiskMB: 20 * 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + nodes = append(nodes, node) + } + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create some low priority batch jobs and allocations for them + // One job uses a reserved port + job1 := mock.BatchJob() + job1.Type = structs.JobTypeBatch + job1.Priority = 20 + job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ + { + MBits: 200, + ReservedPorts: []structs.Port{ { - MBits: 200, - ReservedPorts: []structs.Port{ - { - Label: "web", - Value: 80, - }, - }, + Label: "web", + Value: 80, }, }, - } + }, + }, + } - alloc1 := mock.Alloc() - alloc1.Job = job1 - alloc1.JobID = job1.ID - alloc1.NodeID = nodes[0].ID - alloc1.Name = "my-job[0]" - alloc1.TaskGroup = job1.TaskGroups[0].Name - alloc1.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 200, - }, - }, - }, + alloc1 := mock.Alloc() + alloc1.Job = job1 + alloc1.JobID = job1.ID + alloc1.NodeID = nodes[0].ID + alloc1.Name = "my-job[0]" + alloc1.TaskGroup = job1.TaskGroups[0].Name + alloc1.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, }, - } - - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - job2 := mock.BatchJob() - job2.Type = structs.JobTypeBatch - job2.Priority = 20 - job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, Networks: []*structs.NetworkResource{ { - MBits: 200, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 200, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } - alloc2 := mock.Alloc() - alloc2.Job = job2 - alloc2.JobID = job2.ID - alloc2.NodeID = nodes[0].ID - alloc2.Name = "my-job[2]" - alloc2.TaskGroup = job2.TaskGroups[0].Name - alloc2.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - MBits: 200, - }, - }, - }, + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + job2 := mock.BatchJob() + job2.Type = structs.JobTypeBatch + job2.Priority = 20 + job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ + { + MBits: 200, + }, + }, + } + + alloc2 := mock.Alloc() + alloc2.Job = job2 + alloc2.JobID = job2.ID + alloc2.NodeID = nodes[0].ID + alloc2.Name = "my-job[2]" + alloc2.TaskGroup = job2.TaskGroups[0].Name + alloc2.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - job3 := mock.Job() - job3.Type = structs.JobTypeBatch - job3.Priority = 40 - job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, Networks: []*structs.NetworkResource{ { Device: "eth0", - MBits: 400, + IP: "192.168.0.100", + MBits: 200, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + job3 := mock.Job() + job3.Type = structs.JobTypeBatch + job3.Priority = 40 + job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + MBits: 400, + }, + }, + } - alloc3 := mock.Alloc() - alloc3.Job = job3 - alloc3.JobID = job3.ID - alloc3.NodeID = nodes[0].ID - alloc3.Name = "my-job[0]" - alloc3.TaskGroup = job3.TaskGroups[0].Name - alloc3.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 25, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 400, - }, - }, - }, + alloc3 := mock.Alloc() + alloc3.Job = job3 + alloc3.JobID = job3.ID + alloc3.NodeID = nodes[0].ID + alloc3.Name = "my-job[0]" + alloc3.TaskGroup = job3.TaskGroups[0].Name + alloc3.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 25, }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) - - // Create a high priority job and allocs for it - // These allocs should not be preempted - - job4 := mock.BatchJob() - job4.Type = structs.JobTypeBatch - job4.Priority = 100 - job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, Networks: []*structs.NetworkResource{ { - MBits: 100, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 400, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) + + // Create a high priority job and allocs for it + // These allocs should not be preempted + + job4 := mock.BatchJob() + job4.Type = structs.JobTypeBatch + job4.Priority = 100 + job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, + Networks: []*structs.NetworkResource{ + { + MBits: 100, + }, + }, + } - alloc4 := mock.Alloc() - alloc4.Job = job4 - alloc4.JobID = job4.ID - alloc4.NodeID = nodes[0].ID - alloc4.Name = "my-job4[0]" - alloc4.TaskGroup = job4.TaskGroups[0].Name - alloc4.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2048, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 100, - }, - }, - }, + alloc4 := mock.Alloc() + alloc4.Job = job4 + alloc4.JobID = job4.ID + alloc4.NodeID = nodes[0].ID + alloc4.Name = "my-job4[0]" + alloc4.TaskGroup = job4.TaskGroups[0].Name + alloc4.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 2 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 2048, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) - - // Create a system job such that it would need to preempt both allocs to succeed - job := mock.SystemJob() - job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1948, - MemoryMB: 256, Networks: []*structs.NetworkResource{ { - MBits: 800, - DynamicPorts: []structs.Port{{Label: "http"}}, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 100, }, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - require := require.New(t) - require.Nil(err) - - // Ensure a single plan - require.Equal(1, len(h.Plans)) - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - require.Nil(plan.Annotations) - - // Ensure the plan allocated on both nodes - var planned []*structs.Allocation - preemptingAllocId := "" - require.Equal(2, len(plan.NodeAllocation)) - - // The alloc that got placed on node 1 is the preemptor - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - for _, alloc := range allocList { - if alloc.NodeID == nodes[0].ID { - preemptingAllocId = alloc.ID - } - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 2 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) + + // Create a system job such that it would need to preempt both allocs to succeed + job := mock.SystemJob() + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1948, + MemoryMB: 256, + Networks: []*structs.NetworkResource{ + { + MBits: 800, + DynamicPorts: []structs.Port{{Label: "http"}}, + }, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure all allocations placed - require.Equal(2, len(out)) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + require := require.New(t) + require.Nil(err) - // Verify that one node has preempted allocs - require.NotNil(plan.NodePreemptions[nodes[0].ID]) - preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + // Ensure a single plan + require.Equal(1, len(h.Plans)) + plan := h.Plans[0] - // Verify that three jobs have preempted allocs - require.Equal(3, len(preemptedAllocs)) + // Ensure the plan doesn't have annotations. + require.Nil(plan.Annotations) - expectedPreemptedAllocIDs := []string{alloc1.ID, alloc2.ID, alloc3.ID} + // Ensure the plan allocated on both nodes + var planned []*structs.Allocation + preemptingAllocId := "" + require.Equal(2, len(plan.NodeAllocation)) - // We expect job1, job2 and job3 to have preempted allocations - // job4 should not have any allocs preempted - for _, alloc := range preemptedAllocs { - require.Contains(expectedPreemptedAllocIDs, alloc.ID) - } - // Look up the preempted allocs by job ID - ws = memdb.NewWatchSet() - - for _, allocID := range expectedPreemptedAllocIDs { - evictedAlloc, err := h.State.AllocByID(ws, allocID) - noErr(t, err) - require.Equal(structs.AllocDesiredStatusEvict, evictedAlloc.DesiredStatus) - require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), evictedAlloc.DesiredDescription) + // The alloc that got placed on node 1 is the preemptor + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + for _, alloc := range allocList { + if alloc.NodeID == nodes[0].ID { + preemptingAllocId = alloc.ID } + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + require.Equal(2, len(out)) + + // Verify that one node has preempted allocs + require.NotNil(plan.NodePreemptions[nodes[0].ID]) + preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + + // Verify that three jobs have preempted allocs + require.Equal(3, len(preemptedAllocs)) + + expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} + + // We expect job1, job2 and job3 to have preempted allocations + // job4 should not have any allocs preempted + for _, alloc := range preemptedAllocs { + require.Contains(expectedPreemptedJobIDs, alloc.JobID) } + // Look up the preempted allocs by job ID + ws = memdb.NewWatchSet() + + for _, jobId := range expectedPreemptedJobIDs { + out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) + noErr(t, err) + for _, alloc := range out { + require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) + require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + } diff --git a/scheduler/testing.go b/scheduler/testing.go index 6eb43794572..876ff101db5 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/mitchellh/go-testing-interface" + testing "github.com/mitchellh/go-testing-interface" "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" @@ -54,29 +54,27 @@ type Harness struct { nextIndex uint64 nextIndexLock sync.Mutex - allowPlanOptimization bool + optimizePlan bool } // NewHarness is used to make a new testing harness -func NewHarness(t testing.T, allowPlanOptimization bool) *Harness { +func NewHarness(t testing.T) *Harness { state := state.TestStateStore(t) h := &Harness{ - t: t, - State: state, - nextIndex: 1, - allowPlanOptimization: allowPlanOptimization, + t: t, + State: state, + nextIndex: 1, } return h } // NewHarnessWithState creates a new harness with the given state for testing // purposes. -func NewHarnessWithState(t testing.T, state *state.StateStore, allowPlanOptimization bool) *Harness { +func NewHarnessWithState(t testing.T, state *state.StateStore) *Harness { return &Harness{ - t: t, - State: state, - nextIndex: 1, - allowPlanOptimization: allowPlanOptimization, + t: t, + State: state, + nextIndex: 1, } } @@ -137,7 +135,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er NodePreemptions: preemptedAllocs, } - if h.allowPlanOptimization { + if h.optimizePlan { req.AllocsStopped = allocsStopped req.AllocsUpdated = allocsUpdated } else { @@ -154,6 +152,12 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er return result, nil, err } +// OptimizePlan is a function used only for Harness to help set the optimzePlan field, +// since Harness doesn't have access to a Server object +func (h *Harness) OptimizePlan(optimize bool) { + h.optimizePlan = optimize +} + func updateCreateTimestamp(allocations []*structs.Allocation, now int64) { // Set the time the alloc was applied for the first time. This can be used // to approximate the scheduling time. @@ -234,15 +238,15 @@ func (h *Harness) Snapshot() State { // Scheduler is used to return a new scheduler from // a snapshot of current state using the harness for planning. -func (h *Harness) Scheduler(factory Factory, allowPlanOptimization bool) Scheduler { +func (h *Harness) Scheduler(factory Factory) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h, allowPlanOptimization) + return factory(logger, h.Snapshot(), h) } // Process is used to process an evaluation given a factory // function to create the scheduler func (h *Harness) Process(factory Factory, eval *structs.Evaluation) error { - sched := h.Scheduler(factory, h.allowPlanOptimization) + sched := h.Scheduler(factory) return sched.Process(eval) } diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 13864b43214..08f5812aac6 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -621,7 +621,7 @@ func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { } func TestSetStatus(t *testing.T) { - h := NewHarness(t, true) + h := NewHarness(t) logger := testlog.HCLogger(t) eval := mock.Eval() status := "a" @@ -640,7 +640,7 @@ func TestSetStatus(t *testing.T) { } // Test next evals - h = NewHarness(t, true) + h = NewHarness(t) next := mock.Eval() if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -656,7 +656,7 @@ func TestSetStatus(t *testing.T) { } // Test blocked evals - h = NewHarness(t, true) + h = NewHarness(t) blocked := mock.Eval() if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -672,7 +672,7 @@ func TestSetStatus(t *testing.T) { } // Test metrics - h = NewHarness(t, true) + h = NewHarness(t) metrics := map[string]*structs.AllocMetric{"foo": nil} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -688,7 +688,7 @@ func TestSetStatus(t *testing.T) { } // Test queued allocations - h = NewHarness(t, true) + h = NewHarness(t) queuedAllocs := map[string]int{"web": 1} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { @@ -704,7 +704,7 @@ func TestSetStatus(t *testing.T) { t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) } - h = NewHarness(t, true) + h = NewHarness(t) dID := uuid.Generate() if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { t.Fatalf("setStatus() failed: %v", err)