From 4eedab18a75a4545a853c7668a8ef13d3fe507be Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Mon, 4 Mar 2019 01:49:32 -0800 Subject: [PATCH 1/7] Add code for plan normalization --- nomad/fsm.go | 9 +- nomad/job_endpoint.go | 3 +- nomad/leader.go | 4 +- nomad/operator_endpoint.go | 4 +- nomad/plan_apply.go | 139 +++++++++++++++++++++--------- nomad/server.go | 4 +- nomad/state/state_store.go | 172 ++++++++++++++++++++++++++----------- nomad/structs/structs.go | 71 ++++++++++++--- nomad/util.go | 9 +- nomad/util_test.go | 2 +- nomad/worker.go | 6 +- nomad/worker_test.go | 2 +- scheduler/generic_sched.go | 32 ++++--- scheduler/scheduler.go | 6 +- scheduler/system_sched.go | 23 +++-- scheduler/testing.go | 2 +- scheduler/util.go | 9 +- 17 files changed, 349 insertions(+), 148 deletions(-) diff --git a/nomad/fsm.go b/nomad/fsm.go index 3c91b8f5c69..fc3cebad1b4 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -86,6 +86,9 @@ type nomadFSM struct { // new state store). Everything internal here is synchronized by the // Raft side, so doesn't need to lock this. stateLock sync.RWMutex + + // Reference to the server the FSM is running on + server *Server } // nomadSnapshot is used to provide a snapshot of the current @@ -121,7 +124,7 @@ type FSMConfig struct { } // NewFSMPath is used to construct a new FSM with a blank state -func NewFSM(config *FSMConfig) (*nomadFSM, error) { +func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { // Create a state store sconfig := &state.StateStoreConfig{ Logger: config.Logger, @@ -142,6 +145,7 @@ func NewFSM(config *FSMConfig) (*nomadFSM, error) { timetable: NewTimeTable(timeTableGranularity, timeTableLimit), enterpriseAppliers: make(map[structs.MessageType]LogApplier, 8), enterpriseRestorers: make(map[SnapshotType]SnapshotRestorer, 8), + server: server, } // Register all the log applier functions @@ -1423,7 +1427,8 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner) + allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanDenormalization, true) + sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) if err != nil { return err } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 1a5cdaf4300..332c048a4c7 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1216,7 +1216,8 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner) + allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanDenormalization, true) + sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) if err != nil { return err } diff --git a/nomad/leader.go b/nomad/leader.go index 8e26a56f514..17c8199962f 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -1243,7 +1243,7 @@ func (s *Server) getOrCreateAutopilotConfig() *structs.AutopilotConfig { return config } - if !ServersMeetMinimumVersion(s.Members(), minAutopilotVersion) { + if !ServersMeetMinimumVersion(s.Members(), minAutopilotVersion, false) { s.logger.Named("autopilot").Warn("can't initialize until all servers are above minimum version", "min_version", minAutopilotVersion) return nil } @@ -1270,7 +1270,7 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration { if config != nil { return config } - if !ServersMeetMinimumVersion(s.Members(), minSchedulerConfigVersion) { + if !ServersMeetMinimumVersion(s.Members(), minSchedulerConfigVersion, false) { s.logger.Named("core").Warn("can't initialize scheduler config until all servers are above minimum version", "min_version", minSchedulerConfigVersion) return nil } diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go index fc9edabdd97..c44e14c31c3 100644 --- a/nomad/operator_endpoint.go +++ b/nomad/operator_endpoint.go @@ -237,7 +237,7 @@ func (op *Operator) AutopilotSetConfiguration(args *structs.AutopilotSetConfigRe } // All servers should be at or above 0.8.0 to apply this operatation - if !ServersMeetMinimumVersion(op.srv.Members(), minAutopilotVersion) { + if !ServersMeetMinimumVersion(op.srv.Members(), minAutopilotVersion, false) { return fmt.Errorf("All servers should be running version %v to update autopilot config", minAutopilotVersion) } @@ -305,7 +305,7 @@ func (op *Operator) SchedulerSetConfiguration(args *structs.SchedulerSetConfigRe } // All servers should be at or above 0.9.0 to apply this operatation - if !ServersMeetMinimumVersion(op.srv.Members(), minSchedulerConfigVersion) { + if !ServersMeetMinimumVersion(op.srv.Members(), minSchedulerConfigVersion, false) { return fmt.Errorf("All servers should be running version %v to update scheduler config", minSchedulerConfigVersion) } // Apply the update diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index f40690d00f2..dc991323288 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -15,7 +15,7 @@ import ( "github.com/hashicorp/raft" ) -// planner is used to mange the submitted allocation plans that are waiting +// planner is used to manage the submitted allocation plans that are waiting // to be accessed by the leader type planner struct { *Server @@ -149,52 +149,90 @@ func (p *planner) planApply() { // applyPlan is used to apply the plan result and to return the alloc index func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap *state.StateSnapshot) (raft.ApplyFuture, error) { - // Determine the minimum number of updates, could be more if there - // are multiple updates per node - minUpdates := len(result.NodeUpdate) - minUpdates += len(result.NodeAllocation) - // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: plan.Job, - Alloc: make([]*structs.Allocation, 0, minUpdates), + Job: plan.Job, }, Deployment: result.Deployment, DeploymentUpdates: result.DeploymentUpdates, EvalID: plan.EvalID, NodePreemptions: make([]*structs.Allocation, 0, len(result.NodePreemptions)), } - for _, updateList := range result.NodeUpdate { - req.Alloc = append(req.Alloc, updateList...) - } - for _, allocList := range result.NodeAllocation { - req.Alloc = append(req.Alloc, allocList...) - } - for _, preemptions := range result.NodePreemptions { - req.NodePreemptions = append(req.NodePreemptions, preemptions...) - } - - // Set the time the alloc was applied for the first time. This can be used - // to approximate the scheduling time. + preemptedJobIDs := make(map[structs.NamespacedID]struct{}) now := time.Now().UTC().UnixNano() - for _, alloc := range req.Alloc { - if alloc.CreateTime == 0 { - alloc.CreateTime = now + + if ServersMeetMinimumVersion(p.Members(), MinVersionPlanDenormalization, true) { + // Initialize the allocs request using the new optimized log entry format. + // Determine the minimum number of updates, could be more if there + // are multiple updates per node + req.AllocsStopped = make([]*structs.Allocation, 0, len(result.NodeUpdate)) + req.AllocsUpdated = make([]*structs.Allocation, 0, len(result.NodeAllocation)) + + for _, updateList := range result.NodeUpdate { + for _, stoppedAlloc := range updateList { + req.AllocsStopped = append(req.AllocsStopped, &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: stoppedAlloc.DesiredDescription, + ClientStatus: stoppedAlloc.ClientStatus, + ModifyTime: now, + }) + } } - alloc.ModifyTime = now - } - // Set modify time for preempted allocs if any - // Also gather jobids to create follow up evals - preemptedJobIDs := make(map[structs.NamespacedID]struct{}) - for _, alloc := range req.NodePreemptions { - alloc.ModifyTime = now - id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} - _, ok := preemptedJobIDs[id] - if !ok { - preemptedJobIDs[id] = struct{}{} + for _, allocList := range result.NodeAllocation { + req.AllocsUpdated = append(req.AllocsUpdated, allocList...) + } + + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + updateAllocTimestamps(req.AllocsUpdated, now) + + for _, preemptions := range result.NodePreemptions { + for _, preemptedAlloc := range preemptions { + req.NodePreemptions = append(req.NodePreemptions, &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, + ModifyTime: now, + }) + + // Gather jobids to create follow up evals + appendNamespacedJobID(preemptedJobIDs, preemptedAlloc) + } + } + } else { + // Deprecated: This code path is deprecated and will only be used to support + // application of older log entries. Expected to be removed in a future version. + + // Determine the minimum number of updates, could be more if there + // are multiple updates per node + minUpdates := len(result.NodeUpdate) + minUpdates += len(result.NodeAllocation) + + // Initialize the allocs request using the older log entry format + req.Alloc = make([]*structs.Allocation, 0, minUpdates) + + for _, updateList := range result.NodeUpdate { + req.Alloc = append(req.Alloc, updateList...) + } + for _, allocList := range result.NodeAllocation { + req.Alloc = append(req.Alloc, allocList...) + } + + for _, preemptions := range result.NodePreemptions { + req.NodePreemptions = append(req.NodePreemptions, preemptions...) + } + + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + updateAllocTimestamps(req.Alloc, now) + + // Set modify time for preempted allocs if any + // Also gather jobids to create follow up evals + for _, alloc := range req.NodePreemptions { + alloc.ModifyTime = now + appendNamespacedJobID(preemptedJobIDs, alloc) } } @@ -232,6 +270,22 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap return future, nil } +func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *structs.Allocation) { + id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} + if _, ok := jobIDs[id]; !ok { + jobIDs[id] = struct{}{} + } +} + +func updateAllocTimestamps(allocations []*structs.Allocation, timestamp int64) { + for _, alloc := range allocations { + if alloc.CreateTime == 0 { + alloc.CreateTime = timestamp + } + alloc.ModifyTime = timestamp + } +} + // asyncPlanWait is used to apply and respond to a plan async func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, result *structs.PlanResult, pending *pendingPlan) { @@ -264,6 +318,15 @@ func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, func evaluatePlan(pool *EvaluatePool, snap *state.StateSnapshot, plan *structs.Plan, logger log.Logger) (*structs.PlanResult, error) { defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now()) + err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, plan.Job) + if err != nil { + return nil, err + } + err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, plan.Job) + if err != nil { + return nil, err + } + // Check if the plan exceeds quota overQuota, err := evaluatePlanQuota(snap, plan) if err != nil { @@ -521,15 +584,11 @@ func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID stri // Remove any preempted allocs if preempted := plan.NodePreemptions[nodeID]; len(preempted) > 0 { - for _, allocs := range preempted { - remove = append(remove, allocs) - } + remove = append(remove, preempted...) } if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 { - for _, alloc := range updated { - remove = append(remove, alloc) - } + remove = append(remove, updated...) } proposed := structs.RemoveAllocs(existingAlloc, remove) proposed = append(proposed, plan.NodeAllocation[nodeID]...) diff --git a/nomad/server.go b/nomad/server.go index c35124c510c..f8c4d2e4ca5 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1078,7 +1078,7 @@ func (s *Server) setupRaft() error { Region: s.Region(), } var err error - s.fsm, err = NewFSM(fsmConfig) + s.fsm, err = NewFSM(fsmConfig, s) if err != nil { return err } @@ -1173,7 +1173,7 @@ func (s *Server) setupRaft() error { if err != nil { return fmt.Errorf("recovery failed to parse peers.json: %v", err) } - tmpFsm, err := NewFSM(fsmConfig) + tmpFsm, err := NewFSM(fsmConfig, s) if err != nil { return fmt.Errorf("recovery failed to make temp FSM: %v", err) } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 84201cbaaf0..24b90a2f45b 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -170,6 +170,21 @@ RUN_QUERY: // UpsertPlanResults is used to upsert the results of a plan. func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanResultsRequest) error { + snapshot, err := s.Snapshot() + if err != nil { + return err + } + + err = snapshot.DenormalizeAllocationsSlice(results.AllocsStopped, results.Job) + if err != nil { + return err + } + + err = snapshot.DenormalizeAllocationsSlice(results.NodePreemptions, results.Job) + if err != nil { + return err + } + txn := s.db.Txn(true) defer txn.Abort() @@ -185,34 +200,6 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR s.upsertDeploymentUpdates(index, results.DeploymentUpdates, txn) } - // Attach the job to all the allocations. It is pulled out in the payload to - // avoid the redundancy of encoding, but should be denormalized prior to - // being inserted into MemDB. - structs.DenormalizeAllocationJobs(results.Job, results.Alloc) - - // COMPAT(0.11): Remove in 0.11 - // Calculate the total resources of allocations. It is pulled out in the - // payload to avoid encoding something that can be computed, but should be - // denormalized prior to being inserted into MemDB. - for _, alloc := range results.Alloc { - if alloc.Resources != nil { - continue - } - - alloc.Resources = new(structs.Resources) - for _, task := range alloc.TaskResources { - alloc.Resources.Add(task) - } - - // Add the shared resources - alloc.Resources.Add(alloc.SharedResources) - } - - // Upsert the allocations - if err := s.upsertAllocsImpl(index, results.Alloc, txn); err != nil { - return err - } - // COMPAT: Nomad versions before 0.7.1 did not include the eval ID when // applying the plan. Thus while we are upgrading, we ignore updating the // modify index of evaluations from older plans. @@ -223,35 +210,33 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR } } - // Prepare preempted allocs in the plan results for update - var preemptedAllocs []*structs.Allocation - for _, preemptedAlloc := range results.NodePreemptions { - // Look for existing alloc - existing, err := txn.First("allocs", "id", preemptedAlloc.ID) - if err != nil { - return fmt.Errorf("alloc lookup failed: %v", err) - } + noOfAllocs := len(results.NodePreemptions) - // Nothing to do if this does not exist - if existing == nil { - continue - } - exist := existing.(*structs.Allocation) + if len(results.Alloc) > 0 { + // COMPAT 0.11: This branch will be removed, when Alloc is removed + // Attach the job to all the allocations. It is pulled out in the payload to + // avoid the redundancy of encoding, but should be denormalized prior to + // being inserted into MemDB. + addComputedAllocAttrs(results.Alloc, results.Job) + noOfAllocs += len(results.Alloc) + } else { + // Attach the job to all the allocations. It is pulled out in the payload to + // avoid the redundancy of encoding, but should be denormalized prior to + // being inserted into MemDB. + addComputedAllocAttrs(results.AllocsUpdated, results.Job) + noOfAllocs += len(results.AllocsStopped) + len(results.AllocsUpdated) + } - // Copy everything from the existing allocation - copyAlloc := exist.Copy() + allocsToUpsert := make([]*structs.Allocation, 0, noOfAllocs) - // Only update the fields set by the scheduler - copyAlloc.DesiredStatus = preemptedAlloc.DesiredStatus - copyAlloc.PreemptedByAllocation = preemptedAlloc.PreemptedByAllocation - copyAlloc.DesiredDescription = preemptedAlloc.DesiredDescription - copyAlloc.ModifyTime = preemptedAlloc.ModifyTime - preemptedAllocs = append(preemptedAllocs, copyAlloc) + // COMPAT 0.11: This append should be removed when Alloc is removed + allocsToUpsert = append(allocsToUpsert, results.Alloc...) - } + allocsToUpsert = append(allocsToUpsert, results.AllocsStopped...) + allocsToUpsert = append(allocsToUpsert, results.AllocsUpdated...) + allocsToUpsert = append(allocsToUpsert, results.NodePreemptions...) - // Upsert the preempted allocations - if err := s.upsertAllocsImpl(index, preemptedAllocs, txn); err != nil { + if err := s.upsertAllocsImpl(index, allocsToUpsert, txn); err != nil { return err } @@ -266,6 +251,28 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR return nil } +func addComputedAllocAttrs(allocs []*structs.Allocation, job *structs.Job) { + structs.DenormalizeAllocationJobs(job, allocs) + + // COMPAT(0.11): Remove in 0.11 + // Calculate the total resources of allocations. It is pulled out in the + // payload to avoid encoding something that can be computed, but should be + // denormalized prior to being inserted into MemDB. + for _, alloc := range allocs { + if alloc.Resources != nil { + continue + } + + alloc.Resources = new(structs.Resources) + for _, task := range alloc.TaskResources { + alloc.Resources.Add(task) + } + + // Add the shared resources + alloc.Resources.Add(alloc.SharedResources) + } +} + // upsertDeploymentUpdates updates the deployments given the passed status // updates. func (s *StateStore) upsertDeploymentUpdates(index uint64, updates []*structs.DeploymentStatusUpdate, txn *memdb.Txn) error { @@ -4100,6 +4107,67 @@ type StateSnapshot struct { StateStore } +// DenormalizeAllocationsMap takes in a map of nodes to allocations, and queries the +// Allocation for each of the Allocation diffs and merges the updated attributes with +// the existing Allocation, and attaches the Job provided +func (s *StateSnapshot) DenormalizeAllocationsMap(nodeAllocations map[string][]*structs.Allocation, job *structs.Job) error { + for _, allocDiffs := range nodeAllocations { + if err := s.DenormalizeAllocationsSlice(allocDiffs, job); err != nil { + return err + } + } + return nil +} + +// DenormalizeAllocationsSlice queries the Allocation for each of the Allocation diffs and merges +// the updated attributes with the existing Allocation, and attaches the Job provided +func (s *StateSnapshot) DenormalizeAllocationsSlice(allocDiffs []*structs.Allocation, job *structs.Job) error { + // Output index for denormalized Allocations + j := 0 + + for _, allocDiff := range allocDiffs { + alloc, err := s.AllocByID(nil, allocDiff.ID) + if err != nil { + return fmt.Errorf("alloc lookup failed: %v", err) + } + if alloc == nil { + continue + } + + // Merge the updates to the Allocation + allocCopy := alloc.CopySkipJob() + allocCopy.Job = job + + if allocDiff.PreemptedByAllocation != "" { + // If alloc is a preemption + allocCopy.PreemptedByAllocation = allocDiff.PreemptedByAllocation + allocCopy.DesiredDescription = getPreemptedAllocDesiredDescription(allocDiff.PreemptedByAllocation) + allocCopy.DesiredStatus = structs.AllocDesiredStatusEvict + } else { + // If alloc is a stopped alloc + allocCopy.DesiredDescription = allocDiff.DesiredDescription + allocCopy.DesiredStatus = structs.AllocDesiredStatusStop + if allocDiff.ClientStatus != "" { + allocCopy.ClientStatus = allocDiff.ClientStatus + } + } + if allocDiff.ModifyTime != 0 { + allocCopy.ModifyTime = allocDiff.ModifyTime + } + + // Update the allocDiff in the slice to equal the denormalized alloc + allocDiffs[j] = allocCopy + j++ + } + // Retain only the denormalized Allocations in the slice + allocDiffs = allocDiffs[:j] + return nil +} + +func getPreemptedAllocDesiredDescription(PreemptedByAllocID string) string { + return fmt.Sprintf("Preempted by alloc ID %v", PreemptedByAllocID) +} + // StateRestore is used to optimize the performance when // restoring state by only using a single large transaction // instead of thousands of sub transactions diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 48d7b57d9af..2d7614dbdc1 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -28,8 +28,8 @@ import ( "github.com/gorhill/cronexpr" "github.com/hashicorp/consul/api" hcodec "github.com/hashicorp/go-msgpack/codec" - multierror "github.com/hashicorp/go-multierror" - version "github.com/hashicorp/go-version" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/args" @@ -660,9 +660,9 @@ type ApplyPlanResultsRequest struct { // the evaluation itself being updated. EvalID string - // NodePreemptions is a slice of allocations from other lower priority jobs + // NodePreemptions is a slice of allocation diffs from other lower priority jobs // that are preempted. Preempted allocations are marked as evicted. - NodePreemptions []*Allocation + NodePreemptions []*AllocationDiff // PreemptionEvals is a slice of follow up evals for jobs whose allocations // have been preempted to place allocs in this plan @@ -674,8 +674,16 @@ type ApplyPlanResultsRequest struct { // within a single transaction type AllocUpdateRequest struct { // Alloc is the list of new allocations to assign + // Deprecated: Replaced with two separate slices, one containing stopped allocations + // and another containing updated allocations Alloc []*Allocation + // Allocations to stop. Contains only the diff, not the entire allocation + AllocsStopped []*AllocationDiff + + // New or updated allocations + AllocsUpdated []*Allocation + // Evals is the list of new evaluations to create // Evals are valid only when used in the Raft RPC Evals []*Evaluation @@ -7168,6 +7176,9 @@ const ( // Allocation is used to allocate the placement of a task group to a node. type Allocation struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // ID of the allocation (UUID) ID string @@ -7282,6 +7293,10 @@ type Allocation struct { ModifyTime int64 } +// AllocationDiff is a type alias for Allocation used to indicate that a diff is +// and not the entire allocation +type AllocationDiff = Allocation + // Index returns the index of the allocation. If the allocation is from a task // group with count greater than 1, there will be multiple allocations for it. func (a *Allocation) Index() uint { @@ -7296,11 +7311,12 @@ func (a *Allocation) Index() uint { return uint(num) } +// Copy provides a copy of the allocation and deep copies the job func (a *Allocation) Copy() *Allocation { return a.copyImpl(true) } -// Copy provides a copy of the allocation but doesn't deep copy the job +// CopySkipJob provides a copy of the allocation but doesn't deep copy the job func (a *Allocation) CopySkipJob() *Allocation { return a.copyImpl(false) } @@ -8037,6 +8053,9 @@ const ( // potentially taking action (allocation of work) or doing nothing if the state // of the world does not require it. type Evaluation struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // ID is a randomly generated UUID used for this evaluation. This // is assigned upon the creation of the evaluation. ID string @@ -8227,7 +8246,7 @@ func (e *Evaluation) ShouldBlock() bool { // MakePlan is used to make a plan from the given evaluation // for a given Job -func (e *Evaluation) MakePlan(j *Job) *Plan { +func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { p := &Plan{ EvalID: e.ID, Priority: e.Priority, @@ -8235,6 +8254,7 @@ func (e *Evaluation) MakePlan(j *Job) *Plan { NodeUpdate: make(map[string][]*Allocation), NodeAllocation: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), + NormalizeAllocs: allowPlanOptimization, } if j != nil { p.AllAtOnce = j.AllAtOnce @@ -8304,6 +8324,9 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { // are submitted to the leader which verifies that resources have // not been overcommitted before admitting the plan. type Plan struct { + // msgpack omit empty fields during serialization + _struct bool `codec:",omitempty"` // nolint: structcheck + // EvalID is the evaluation ID this plan is associated with EvalID string @@ -8353,11 +8376,14 @@ type Plan struct { // lower priority jobs that are preempted. Preempted allocations are marked // as evicted. NodePreemptions map[string][]*Allocation + + // Indicates whether allocs are normalized in the Plan + NormalizeAllocs bool `codec:"-"` } -// AppendUpdate marks the allocation for eviction. The clientStatus of the +// AppendStoppedAlloc marks an allocation to be stopped. The clientStatus of the // allocation may be optionally set by passing in a non-empty value. -func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clientStatus string) { +func (p *Plan) AppendStoppedAlloc(alloc *Allocation, desiredDesc, clientStatus string) { newAlloc := new(Allocation) *newAlloc = *alloc @@ -8373,7 +8399,7 @@ func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clien // Strip the resources as it can be rebuilt. newAlloc.Resources = nil - newAlloc.DesiredStatus = desiredStatus + newAlloc.DesiredStatus = AllocDesiredStatusStop newAlloc.DesiredDescription = desiredDesc if clientStatus != "" { @@ -8387,12 +8413,12 @@ func (p *Plan) AppendUpdate(alloc *Allocation, desiredStatus, desiredDesc, clien // AppendPreemptedAlloc is used to append an allocation that's being preempted to the plan. // To minimize the size of the plan, this only sets a minimal set of fields in the allocation -func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, desiredStatus, preemptingAllocID string) { +func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, preemptingAllocID string) { newAlloc := &Allocation{} newAlloc.ID = alloc.ID newAlloc.JobID = alloc.JobID newAlloc.Namespace = alloc.Namespace - newAlloc.DesiredStatus = desiredStatus + newAlloc.DesiredStatus = AllocDesiredStatusEvict newAlloc.PreemptedByAllocation = preemptingAllocID desiredDesc := fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID) @@ -8445,6 +8471,29 @@ func (p *Plan) IsNoOp() bool { len(p.DeploymentUpdates) == 0 } +func (p *Plan) NormalizeAllocations() { + if p.NormalizeAllocs { + for _, allocs := range p.NodeUpdate { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + DesiredDescription: alloc.DesiredDescription, + ClientStatus: alloc.ClientStatus, + } + } + } + + for _, allocs := range p.NodePreemptions { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: alloc.PreemptedByAllocation, + } + } + } + } +} + // PlanResult is the result of a plan submitted to the leader. type PlanResult struct { // NodeUpdate contains all the updates that were committed. diff --git a/nomad/util.go b/nomad/util.go index 44b7119242b..8a6ff407f73 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -14,6 +14,11 @@ import ( "github.com/hashicorp/serf/serf" ) +// MinVersionPlanDenormalization is the minimum version to support the +// denormalization of Plan in SubmitPlan, and the raft log entry committed +// in ApplyPlanResultsRequest +var MinVersionPlanDenormalization = version.Must(version.NewVersion("0.9.1")) + // ensurePath is used to make sure a path exists func ensurePath(path string, dir bool) error { if !dir { @@ -145,9 +150,9 @@ func isNomadServer(m serf.Member) (bool, *serverParts) { // ServersMeetMinimumVersion returns whether the given alive servers are at least on the // given Nomad version -func ServersMeetMinimumVersion(members []serf.Member, minVersion *version.Version) bool { +func ServersMeetMinimumVersion(members []serf.Member, minVersion *version.Version, checkFailedServers bool) bool { for _, member := range members { - if valid, parts := isNomadServer(member); valid && parts.Status == serf.StatusAlive { + if valid, parts := isNomadServer(member); valid && (parts.Status == serf.StatusAlive || (checkFailedServers && parts.Status == serf.StatusFailed)) { // Check if the versions match - version.LessThan will return true for // 0.8.0-rc1 < 0.8.0, so we want to ignore the metadata versionsMatch := slicesMatch(minVersion.Segments(), parts.Build.Segments()) diff --git a/nomad/util_test.go b/nomad/util_test.go index 4fe670cdc71..5614067609c 100644 --- a/nomad/util_test.go +++ b/nomad/util_test.go @@ -162,7 +162,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { } for _, tc := range cases { - result := ServersMeetMinimumVersion(tc.members, tc.ver) + result := ServersMeetMinimumVersion(tc.members, tc.ver, false) if result != tc.expected { t.Fatalf("bad: %v, %v, %v", result, tc.ver.String(), tc) } diff --git a/nomad/worker.go b/nomad/worker.go index f0aefb62d54..6cd8e0107fc 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -284,7 +284,8 @@ func (w *Worker) invokeScheduler(eval *structs.Evaluation, token string) error { if eval.Type == structs.JobTypeCore { sched = NewCoreScheduler(w.srv, snap) } else { - sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w) + allowPlanOptimization := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanDenormalization, true) + sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w, allowPlanOptimization) if err != nil { return fmt.Errorf("failed to instantiate scheduler: %v", err) } @@ -310,6 +311,9 @@ func (w *Worker) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, scheduler. // Add the evaluation token to the plan plan.EvalToken = w.evalToken + // Normalize stopped and preempted allocs before RPC + plan.NormalizeAllocations() + // Setup the request req := structs.PlanRequest{ Plan: plan, diff --git a/nomad/worker_test.go b/nomad/worker_test.go index 2f3e3172831..b7a9e526a21 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -36,7 +36,7 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { } func init() { - scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { + scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { n := &NoopScheduler{ state: s, planner: p, diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index 6c518f528f5..b69b31a64be 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -77,6 +77,10 @@ type GenericScheduler struct { planner Planner batch bool + // Temporary flag introduced till the code for sending/committing full allocs in the Plan can + // be safely removed + allowPlanOptimization bool + eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -94,23 +98,25 @@ type GenericScheduler struct { } // NewServiceScheduler is a factory function to instantiate a new service scheduler -func NewServiceScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewServiceScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { s := &GenericScheduler{ - logger: logger.Named("service_sched"), - state: state, - planner: planner, - batch: false, + logger: logger.Named("service_sched"), + state: state, + planner: planner, + batch: false, + allowPlanOptimization: allowPlanOptimization, } return s } // NewBatchScheduler is a factory function to instantiate a new batch scheduler -func NewBatchScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewBatchScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { s := &GenericScheduler{ - logger: logger.Named("batch_sched"), - state: state, - planner: planner, - batch: true, + logger: logger.Named("batch_sched"), + state: state, + planner: planner, + batch: true, + allowPlanOptimization: allowPlanOptimization, } return s } @@ -224,7 +230,7 @@ func (s *GenericScheduler) process() (bool, error) { s.followUpEvals = nil // Create a plan - s.plan = s.eval.MakePlan(s.job) + s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) if !s.batch { // Get any existing deployment @@ -366,7 +372,7 @@ func (s *GenericScheduler) computeJobAllocs() error { // Handle the stop for _, stop := range results.stop { - s.plan.AppendUpdate(stop.alloc, structs.AllocDesiredStatusStop, stop.statusDescription, stop.clientStatus) + s.plan.AppendStoppedAlloc(stop.alloc, stop.statusDescription, stop.clientStatus) } // Handle the in-place updates @@ -464,7 +470,7 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul stopPrevAlloc, stopPrevAllocDesc := missing.StopPreviousAlloc() prevAllocation := missing.PreviousAllocation() if stopPrevAlloc { - s.plan.AppendUpdate(prevAllocation, structs.AllocDesiredStatusStop, stopPrevAllocDesc, "") + s.plan.AppendStoppedAlloc(prevAllocation, stopPrevAllocDesc, "") } // Compute penalty nodes for rescheduled allocs diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 639b2b8cf28..76a0a69408c 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -28,7 +28,7 @@ var BuiltinSchedulers = map[string]Factory{ // NewScheduler is used to instantiate and return a new scheduler // given the scheduler name, initial state, and planner. -func NewScheduler(name string, logger log.Logger, state State, planner Planner) (Scheduler, error) { +func NewScheduler(name string, logger log.Logger, state State, planner Planner, allowPlanOptimization bool) (Scheduler, error) { // Lookup the factory function factory, ok := BuiltinSchedulers[name] if !ok { @@ -36,12 +36,12 @@ func NewScheduler(name string, logger log.Logger, state State, planner Planner) } // Instantiate the scheduler - sched := factory(logger, state, planner) + sched := factory(logger, state, planner, allowPlanOptimization) return sched, nil } // Factory is used to instantiate a new Scheduler -type Factory func(log.Logger, State, Planner) Scheduler +type Factory func(log.Logger, State, Planner, bool) Scheduler // Scheduler is the top level instance for a scheduler. A scheduler is // meant to only encapsulate business logic, pushing the various plumbing diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index bcfd6e5be3d..2cefcba3478 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -23,6 +23,10 @@ type SystemScheduler struct { state State planner Planner + // Temporary flag introduced till the code for sending/committing full allocs in the Plan can + // be safely removed + allowPlanOptimization bool + eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -41,11 +45,12 @@ type SystemScheduler struct { // NewSystemScheduler is a factory function to instantiate a new system // scheduler. -func NewSystemScheduler(logger log.Logger, state State, planner Planner) Scheduler { +func NewSystemScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { return &SystemScheduler{ - logger: logger.Named("system_sched"), - state: state, - planner: planner, + logger: logger.Named("system_sched"), + state: state, + planner: planner, + allowPlanOptimization: allowPlanOptimization, } } @@ -110,7 +115,7 @@ func (s *SystemScheduler) process() (bool, error) { } // Create a plan - s.plan = s.eval.MakePlan(s.job) + s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) // Reset the failed allocations s.failedTGAllocs = nil @@ -210,18 +215,18 @@ func (s *SystemScheduler) computeJobAllocs() error { // Add all the allocs to stop for _, e := range diff.stop { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocNotNeeded, "") + s.plan.AppendStoppedAlloc(e.Alloc, allocNotNeeded, "") } // Add all the allocs to migrate for _, e := range diff.migrate { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocNodeTainted, "") + s.plan.AppendStoppedAlloc(e.Alloc, allocNodeTainted, "") } // Lost allocations should be transitioned to desired status stop and client // status lost. for _, e := range diff.lost { - s.plan.AppendUpdate(e.Alloc, structs.AllocDesiredStatusStop, allocLost, structs.AllocClientStatusLost) + s.plan.AppendStoppedAlloc(e.Alloc, allocLost, structs.AllocClientStatusLost) } // Attempt to do the upgrades in place @@ -351,7 +356,7 @@ func (s *SystemScheduler) computePlacements(place []allocTuple) error { if option.PreemptedAllocs != nil { var preemptedAllocIDs []string for _, stop := range option.PreemptedAllocs { - s.plan.AppendPreemptedAlloc(stop, structs.AllocDesiredStatusEvict, alloc.ID) + s.plan.AppendPreemptedAlloc(stop, alloc.ID) preemptedAllocIDs = append(preemptedAllocIDs, stop.ID) if s.eval.AnnotatePlan && s.plan.Annotations != nil { diff --git a/scheduler/testing.go b/scheduler/testing.go index f0501010246..61fa7f79c61 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -216,7 +216,7 @@ func (h *Harness) Snapshot() State { // a snapshot of current state using the harness for planning. func (h *Harness) Scheduler(factory Factory) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h) + return factory(logger, h.Snapshot(), h, false) } // Process is used to process an evaluation given a factory diff --git a/scheduler/util.go b/scheduler/util.go index 5f62d31b378..3b9b437de95 100644 --- a/scheduler/util.go +++ b/scheduler/util.go @@ -507,8 +507,7 @@ func inplaceUpdate(ctx Context, eval *structs.Evaluation, job *structs.Job, // the current allocation is discounted when checking for feasibility. // Otherwise we would be trying to fit the tasks current resources and // updated resources. After select is called we can remove the evict. - ctx.Plan().AppendUpdate(update.Alloc, structs.AllocDesiredStatusStop, - allocInPlace, "") + ctx.Plan().AppendStoppedAlloc(update.Alloc, allocInPlace, "") // Attempt to match the task group option := stack.Select(update.TaskGroup, nil) // This select only looks at one node so we don't pass selectOptions @@ -573,7 +572,7 @@ func evictAndPlace(ctx Context, diff *diffResult, allocs []allocTuple, desc stri n := len(allocs) for i := 0; i < n && i < *limit; i++ { a := allocs[i] - ctx.Plan().AppendUpdate(a.Alloc, structs.AllocDesiredStatusStop, desc, "") + ctx.Plan().AppendStoppedAlloc(a.Alloc, desc, "") diff.place = append(diff.place, a) } if n <= *limit { @@ -734,7 +733,7 @@ func updateNonTerminalAllocsToLost(plan *structs.Plan, tainted map[string]*struc if alloc.DesiredStatus == structs.AllocDesiredStatusStop && (alloc.ClientStatus == structs.AllocClientStatusRunning || alloc.ClientStatus == structs.AllocClientStatusPending) { - plan.AppendUpdate(alloc, structs.AllocDesiredStatusStop, allocLost, structs.AllocClientStatusLost) + plan.AppendStoppedAlloc(alloc, allocLost, structs.AllocClientStatusLost) } } } @@ -784,7 +783,7 @@ func genericAllocUpdateFn(ctx Context, stack Stack, evalID string) allocUpdateTy // the current allocation is discounted when checking for feasibility. // Otherwise we would be trying to fit the tasks current resources and // updated resources. After select is called we can remove the evict. - ctx.Plan().AppendUpdate(existing, structs.AllocDesiredStatusStop, allocInPlace, "") + ctx.Plan().AppendStoppedAlloc(existing, allocInPlace, "") // Attempt to match the task group option := stack.Select(newTG, nil) // This select only looks at one node so we don't pass selectOptions From f75c6b4bdb0dbc4e3c1f37cd06f5e70a0d450b81 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Tue, 5 Mar 2019 13:41:41 -0800 Subject: [PATCH 2/7] Add tests for plan normalization --- nomad/fsm_test.go | 2 +- nomad/plan_apply_test.go | 150 + nomad/state/state_store_test.go | 101 +- nomad/structs/structs_test.go | 147 +- nomad/util_test.go | 90 +- nomad/worker_test.go | 68 +- scheduler/generic_sched_test.go | 7282 ++++++++++++++++--------------- scheduler/system_sched_test.go | 3322 +++++++------- scheduler/testing.go | 74 +- scheduler/util_test.go | 12 +- 10 files changed, 6024 insertions(+), 5224 deletions(-) diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 0c3a846a85f..06177a0831d 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -56,7 +56,7 @@ func testFSM(t *testing.T) *nomadFSM { Logger: logger, Region: "global", } - fsm, err := NewFSM(fsmConfig) + fsm, err := NewFSM(fsmConfig, TestServer(t, nil)) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 4dfa7a43b02..619b7b9071d 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -3,6 +3,7 @@ package nomad import ( "reflect" "testing" + "time" memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" @@ -62,6 +63,7 @@ func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { } } +// Deprecated: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { t.Parallel() s1 := TestServer(t, nil) @@ -228,6 +230,154 @@ func TestPlanApply_applyPlan(t *testing.T) { assert.Equal(index, evalOut.ModifyIndex) } +func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.Build = "0.9.1" + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + // Register a fake deployment + oldDeployment := mock.Deployment() + if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { + t.Fatalf("UpsertDeployment failed: %v", err) + } + + // Create a deployment + dnew := mock.Deployment() + + // Create a deployment update for the old deployment id + desiredStatus, desiredStatusDescription := "foo", "bar" + updates := []*structs.DeploymentStatusUpdate{ + { + DeploymentID: oldDeployment.ID, + Status: desiredStatus, + StatusDescription: desiredStatusDescription, + }, + } + + // Register allocs, deployment and deployment update + alloc := mock.Alloc() + stoppedAlloc := mock.Alloc() + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "Desired Description", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) + s1.State().UpsertAllocs(1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + // Create an eval + eval := mock.Eval() + eval.JobID = alloc.JobID + if err := s1.State().UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + timestampBeforeCommit := time.Now().UTC().UnixNano() + planRes := &structs.PlanResult{ + NodeAllocation: map[string][]*structs.Allocation{ + node.ID: {alloc}, + }, + NodeUpdate: map[string][]*structs.Allocation{ + stoppedAlloc.NodeID: {stoppedAllocDiff}, + }, + NodePreemptions: map[string][]*structs.Allocation{ + preemptedAlloc.NodeID: {preemptedAllocDiff}, + }, + Deployment: dnew, + DeploymentUpdates: updates, + } + + // Snapshot the state + snap, err := s1.State().Snapshot() + if err != nil { + t.Fatalf("err: %v", err) + } + + // Create the plan with a deployment + plan := &structs.Plan{ + Job: alloc.Job, + Deployment: dnew, + DeploymentUpdates: updates, + EvalID: eval.ID, + } + + // Apply the plan + future, err := s1.applyPlan(plan, planRes, snap) + assert := assert.New(t) + assert.Nil(err) + + // Verify our optimistic snapshot is updated + ws := memdb.NewWatchSet() + allocOut, err := snap.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + + deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(deploymentOut) + + // Check plan does apply cleanly + index, err := planWaitFuture(future) + assert.Nil(err) + assert.NotEqual(0, index) + + // Lookup the allocation + fsmState := s1.fsm.State() + allocOut, err = fsmState.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + assert.True(allocOut.CreateTime > 0) + assert.True(allocOut.ModifyTime > 0) + assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) + + // Verify stopped alloc diff applied cleanly + updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedStoppedAlloc) + assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) + assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) + assert.Equal(updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) + + // Verify preempted alloc diff applied cleanly + updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedPreemptedAlloc) + assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedPreemptedAlloc.DesiredDescription, + "Preempted by alloc ID " + preemptedAllocDiff.PreemptedByAllocation) + assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) + + // Lookup the new deployment + dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(dout) + + // Lookup the updated deployment + dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) + assert.Nil(err) + assert.NotNil(dout2) + assert.Equal(desiredStatus, dout2.Status) + assert.Equal(desiredStatusDescription, dout2.StatusDescription) + + // Lookup updated eval + evalOut, err := fsmState.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.Equal(index, evalOut.ModifyIndex) +} + func TestPlanApply_EvalPlan_Simple(t *testing.T) { t.Parallel() state := testStateStore(t) diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 3cf3b977606..af248a55729 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -88,6 +88,7 @@ func TestStateStore_Blocking_MinQuery(t *testing.T) { } } +// COMPAT 0.11: Uses AllocUpdateRequest.Alloc // This test checks that: // 1) The job is denormalized // 2) Allocations are created @@ -140,6 +141,94 @@ func TestStateStore_UpsertPlanResults_AllocationsCreated_Denormalized(t *testing assert.EqualValues(1000, evalOut.ModifyIndex) } +// This test checks that: +// 1) The job is denormalized +// 2) Allocations are denormalized and updated with the diff +func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { + state := testStateStore(t) + alloc := mock.Alloc() + job := alloc.Job + alloc.Job = nil + + stoppedAlloc := mock.Alloc() + stoppedAlloc.Job = job + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "desired desc", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAlloc.Job = job + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + + if err := state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc}); err != nil { + t.Fatalf("err: %v", err) + } + + if err := state.UpsertJob(999, job); err != nil { + t.Fatalf("err: %v", err) + } + + eval := mock.Eval() + eval.JobID = job.ID + + // Create an eval + if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + // Create a plan result + res := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + AllocsUpdated: []*structs.Allocation{alloc}, + AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, + Job: job, + }, + EvalID: eval.ID, + NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, + } + assert := assert.New(t) + planModifyIndex := uint64(1000) + err := state.UpsertPlanResults(planModifyIndex, &res) + assert.Nil(err) + + ws := memdb.NewWatchSet() + out, err := state.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.Equal(alloc, out) + + updatedStoppedAlloc, err := state.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.Equal(stoppedAllocDiff.DesiredDescription, updatedStoppedAlloc.DesiredDescription) + assert.Equal(structs.AllocDesiredStatusStop, updatedStoppedAlloc.DesiredStatus) + assert.Equal(stoppedAllocDiff.ClientStatus, updatedStoppedAlloc.ClientStatus) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + + updatedPreemptedAlloc, err := state.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.Equal(structs.AllocDesiredStatusEvict, updatedPreemptedAlloc.DesiredStatus) + assert.Equal(preemptedAllocDiff.PreemptedByAllocation, updatedPreemptedAlloc.PreemptedByAllocation) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + + index, err := state.Index("allocs") + assert.Nil(err) + assert.EqualValues(planModifyIndex, index) + + if watchFired(ws) { + t.Fatalf("bad") + } + + evalOut, err := state.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.EqualValues(planModifyIndex, evalOut.ModifyIndex) +} + // This test checks that the deployment is created and allocations count towards // the deployment func TestStateStore_UpsertPlanResults_Deployment(t *testing.T) { @@ -271,11 +360,9 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { require.NoError(err) minimalPreemptedAlloc := &structs.Allocation{ - ID: preemptedAlloc.ID, - Namespace: preemptedAlloc.Namespace, - DesiredStatus: structs.AllocDesiredStatusEvict, - ModifyTime: time.Now().Unix(), - DesiredDescription: fmt.Sprintf("Preempted by allocation %v", alloc.ID), + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + ModifyTime: time.Now().Unix(), } // Create eval for preempted job @@ -316,7 +403,7 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { preempted, err := state.AllocByID(ws, preemptedAlloc.ID) require.NoError(err) require.Equal(preempted.DesiredStatus, structs.AllocDesiredStatusEvict) - require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by allocation %v", alloc.ID)) + require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by alloc ID %v", alloc.ID)) // Verify eval for preempted job preemptedJobEval, err := state.EvalByID(ws, eval2.ID) diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 90b33b19827..2cd3922aebf 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/hashicorp/consul/api" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" "github.com/kr/pretty" "github.com/stretchr/testify/assert" @@ -2842,6 +2842,151 @@ func TestTaskArtifact_Validate_Checksum(t *testing.T) { } } +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = true + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := &Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDesc, + ClientStatus: AllocClientStatusLost, + } + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsFalse(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = false + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := new(Allocation) + *expectedStoppedAlloc = *stoppedAlloc + expectedStoppedAlloc.DesiredDescription = desiredDesc + expectedStoppedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedStoppedAlloc.ClientStatus = AllocClientStatusLost + expectedStoppedAlloc.Job = nil + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: preemptedAlloc.JobID, + Namespace: preemptedAlloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: preemptedAlloc.AllocatedResources, + TaskResources: preemptedAlloc.TaskResources, + SharedResources: preemptedAlloc.SharedResources, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_AppendStoppedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + } + alloc := MockAlloc() + desiredDesc := "Desired desc" + + plan.AppendStoppedAlloc(alloc, desiredDesc, AllocClientStatusLost) + + appendedAlloc := plan.NodeUpdate[alloc.NodeID][0] + expectedAlloc := new(Allocation) + *expectedAlloc = *alloc + expectedAlloc.DesiredDescription = desiredDesc + expectedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedAlloc.ClientStatus = AllocClientStatusLost + expectedAlloc.Job = nil + assert.Equal(t, expectedAlloc, appendedAlloc) + assert.Equal(t, alloc.Job, plan.Job) +} + +func TestPlan_AppendPreemptedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodePreemptions: make(map[string][]*Allocation), + } + alloc := MockAlloc() + preemptingAllocID := uuid.Generate() + + plan.AppendPreemptedAlloc(alloc, preemptingAllocID) + + appendedAlloc := plan.NodePreemptions[alloc.NodeID][0] + expectedAlloc := &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: alloc.JobID, + Namespace: alloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: alloc.AllocatedResources, + TaskResources: alloc.TaskResources, + SharedResources: alloc.SharedResources, + } + assert.Equal(t, expectedAlloc, appendedAlloc) +} + +func TestPlan_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Plan{}) + + msgPackTags, _ := planType.FieldByName("_struct") + normalizeTag, _ := planType.FieldByName("NormalizeAllocs") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) + assert.Equal(t, normalizeTag.Tag, reflect.StructTag(`codec:"-"`)) +} + +func TestAllocation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Allocation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + +func TestEvaluation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Evaluation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + func TestAllocation_Terminated(t *testing.T) { type desiredState struct { ClientStatus string diff --git a/nomad/util_test.go b/nomad/util_test.go index 5614067609c..f216d5e4243 100644 --- a/nomad/util_test.go +++ b/nomad/util_test.go @@ -86,23 +86,8 @@ func TestIsNomadServer(t *testing.T) { } } -func TestServersMeetMinimumVersion(t *testing.T) { +func TestServersMeetMinimumVersionExcludingFailed(t *testing.T) { t.Parallel() - makeMember := func(version string) serf.Member { - return serf.Member{ - Name: "foo", - Addr: net.IP([]byte{127, 0, 0, 1}), - Tags: map[string]string{ - "role": "nomad", - "region": "aws", - "dc": "east-aws", - "port": "10000", - "build": version, - "vsn": "1", - }, - Status: serf.StatusAlive, - } - } cases := []struct { members []serf.Member @@ -112,7 +97,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, meets reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -120,7 +105,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server in dev, meets reqs { members: []serf.Member{ - makeMember("0.8.5-dev"), + makeMember("0.8.5-dev", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -128,7 +113,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server with meta, meets reqs { members: []serf.Member{ - makeMember("0.7.5+ent"), + makeMember("0.7.5+ent", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -136,16 +121,17 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, doesn't meet reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, }, - // Multiple servers, meets req version + // Multiple servers, meets req version, includes failed that doesn't meet req { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -153,8 +139,8 @@ func TestServersMeetMinimumVersion(t *testing.T) { // Multiple servers, doesn't meet req version { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, @@ -169,6 +155,60 @@ func TestServersMeetMinimumVersion(t *testing.T) { } } +func TestServersMeetMinimumVersionIncludingFailed(t *testing.T) { + t.Parallel() + + cases := []struct { + members []serf.Member + ver *version.Version + expected bool + }{ + // Multiple servers, meets req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.5", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: true, + }, + // Multiple servers, doesn't meet req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: false, + }, + } + + for _, tc := range cases { + result := ServersMeetMinimumVersion(tc.members, tc.ver, true) + if result != tc.expected { + t.Fatalf("bad: %v, %v, %v", result, tc.ver.String(), tc) + } + } +} + +func makeMember(version string, status serf.MemberStatus) serf.Member { + return serf.Member{ + Name: "foo", + Addr: net.IP([]byte{127, 0, 0, 1}), + Tags: map[string]string{ + "role": "nomad", + "region": "aws", + "dc": "east-aws", + "port": "10000", + "build": version, + "vsn": "1", + }, + Status: status, + } +} + func TestShuffleStrings(t *testing.T) { t.Parallel() // Generate input diff --git a/nomad/worker_test.go b/nomad/worker_test.go index b7a9e526a21..b2dc94f22b8 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -8,20 +8,22 @@ import ( "time" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/scheduler" "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/assert" ) type NoopScheduler struct { - state scheduler.State - planner scheduler.Planner - eval *structs.Evaluation - err error + state scheduler.State + planner scheduler.Planner + eval *structs.Evaluation + allowPlanOptimization bool + err error } func (n *NoopScheduler) Process(eval *structs.Evaluation) error { @@ -38,8 +40,9 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { func init() { scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { n := &NoopScheduler{ - state: s, - planner: p, + state: s, + planner: p, + allowPlanOptimization: allowPlanOptimization, } return n } @@ -390,6 +393,57 @@ func TestWorker_SubmitPlan(t *testing.T) { } } +func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 + c.EnabledSchedulers = []string{structs.JobTypeService} + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + job := mock.Job() + eval1 := mock.Eval() + eval1.JobID = job.ID + s1.fsm.State().UpsertJob(0, job) + s1.fsm.State().UpsertEvals(0, []*structs.Evaluation{eval1}) + + stoppedAlloc := mock.Alloc() + preemptedAlloc := mock.Alloc() + s1.fsm.State().UpsertAllocs(5, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + + // Create an allocation plan + plan := &structs.Plan{ + Job: job, + EvalID: eval1.ID, + NodeUpdate: make(map[string][]*structs.Allocation), + NodePreemptions: make(map[string][]*structs.Allocation), + NormalizeAllocs: true, + } + desiredDescription := "desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost) + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + // Attempt to submit a plan + w := &Worker{srv: s1, logger: s1.logger} + w.SubmitPlan(plan) + + assert.Equal(t, &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + }, plan.NodePreemptions[preemptedAlloc.NodeID][0]) + assert.Equal(t, &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDescription, + ClientStatus: structs.AllocClientStatusLost, + }, plan.NodeUpdate[stoppedAlloc.NodeID][0]) +} + func TestWorker_SubmitPlan_MissingNodeRefresh(t *testing.T) { t.Parallel() s1 := TestServer(t, func(c *Config) { diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 70e54d23735..3a4ec76ce58 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -16,614 +16,471 @@ import ( "github.com/stretchr/testify/require" ) -func TestServiceSched_JobRegister(t *testing.T) { - h := NewHarness(t) +func IsPlanOptimizedStr(allowPlanOptimization bool) string { + return fmt.Sprintf("Is plan optimized: %v", allowPlanOptimization) +} - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } +func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a job that uses distinct host and has count 1 higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the eval has no spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure different ports were used. - used := make(map[int]map[string]struct{}) - for _, alloc := range out { - for _, resource := range alloc.TaskResources { - for _, port := range resource.Networks[0].DynamicPorts { - nodeMap, ok := used[port.Value] - if !ok { - nodeMap = make(map[string]struct{}) - used[port.Value] = nodeMap - } - if _, ok := nodeMap[alloc.NodeID]; ok { - t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) - } - nodeMap[alloc.NodeID] = struct{}{} + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} -func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - // Create a job - job := mock.Job() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + rack := "rack2" + if i < 5 { + rack = "rack1" + } + node.Meta["rack"] = rack + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan allocated - plan := h.Plans[0] - planned := make(map[string]*structs.Allocation) - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - planned[alloc.ID] = alloc - } - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 8 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Update the job to force a rolling upgrade - updated := job.Copy() - updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 - noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure we have created only one new allocation - // Ensure a single plan - if len(h1.Plans) != 1 { - t.Fatalf("bad: %#v", h1.Plans) - } - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 10 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocations were placed on the same node as the older - // ones - for _, new := range newPlanned { - if new.PreviousAllocation == "" { - t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - old, ok := planned[new.PreviousAllocation] - if !ok { - t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) - } - if new.NodeID != old.NodeID { - t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) - } - } -} + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] -func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { - h := NewHarness(t) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Create a job with count 2 and disk as 60GB so that only one allocation - // can fit - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 4 { + t.Fatalf("bad: %#v", plan) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure all allocations placed + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure each node was only used twice + used := make(map[string]uint64) + for _, alloc := range out { + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) + } + used[alloc.NodeID]++ + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - // Ensure the eval has a blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + // Create some nodes + for i := 0; i < 2; i++ { + node := mock.Node() + node.Meta["ssd"] = "true" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan allocated only one allocation - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Create a job that uses distinct property only on one task group. + job := mock.Job() + job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.ssd}", + }) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + job.TaskGroups[1].Name = "tg2" + job.TaskGroups[1].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure only one allocation was placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } -func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { - h := NewHarness(t) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a job that uses distinct host and has count 1 higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure all allocations placed + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - plan := h.Plans[0] +} - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { + for _, allowPlanOptimization := range []bool{false, true} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + assert := assert.New(t) - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Create a job that uses distinct property over the node-id + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${node.unique.id}", + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 6; i++ { + node := mock.Node() + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - rack := "rack2" - if i < 5 { - rack = "rack1" - } - node.Meta["rack"] = rack - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 8 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - RTarget: "2", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Create some allocations + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Update the count + job2 := job.Copy() + job2.TaskGroups[0].Count = 6 + assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 4 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] - // Ensure all allocations placed - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") - // Ensure each node was only used twice - used := make(map[string]uint64) - for _, alloc := range out { - if count, _ := used[alloc.NodeID]; count > 2 { - t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) - } - used[alloc.NodeID]++ - } + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + assert.Len(planned, 6, "Planned Allocations") -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { - h := NewHarness(t) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + assert.Nil(err, "AllocsByJob") - // Create some nodes - for i := 0; i < 2; i++ { - node := mock.Node() - node.Meta["ssd"] = "true" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure all allocations placed + assert.Len(out, 6, "Placed Allocations") - // Create a job that uses distinct property only on one task group. - job := mock.Job() - job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.ssd}", + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - - job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") } - - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { - h := NewHarness(t) - assert := assert.New(t) - - // Create a job that uses distinct property over the node-id - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${node.unique.id}", - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 6; i++ { - node := mock.Node() - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - } - - // Create some allocations - var allocs []*structs.Allocation - for i := 0; i < 3; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") - - // Update the count - job2 := job.Copy() - job2.TaskGroups[0].Count = 6 - assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") +// Test job registration with spread configured +func TestServiceSched_Spread(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") + start := uint8(100) + step := uint8(10) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%d%% in dc1", start) + t.Run(name, func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + remaining := uint8(100 - start) + // Create a job that uses spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + SpreadTarget: []*structs.SpreadTarget{ + { + Value: "dc1", + Percent: start, + }, + { + Value: "dc2", + Percent: remaining, + }, + }, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") + + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 10 - i + if i > 0 { + expectedCounts["dc2"] = i + } + require.Equal(t, expectedCounts, dcAllocsMap) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + start = start - step + } + }) } - assert.Len(planned, 6, "Planned Allocations") - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - assert.Nil(err, "AllocsByJob") - - // Ensure all allocations placed - assert.Len(out, 6, "Placed Allocations") - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// Test job registration with spread configured -func TestServiceSched_Spread(t *testing.T) { - assert := assert.New(t) - - start := uint8(100) - step := uint8(10) +// Test job registration with even spread across dc +func TestServiceSched_EvenSpread(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) - for i := 0; i < 10; i++ { - name := fmt.Sprintf("%d%% in dc1", start) - t.Run(name, func(t *testing.T) { - h := NewHarness(t) - remaining := uint8(100 - start) - // Create a job that uses spread over data center + h := NewHarness(t, allowPlanOptimization) + // Create a job that uses even spread over data center job := mock.Job() job.Datacenters = []string{"dc1", "dc2"} job.TaskGroups[0].Count = 10 @@ -631,16 +488,6 @@ func TestServiceSched_Spread(t *testing.T) { &structs.Spread{ Attribute: "${node.datacenter}", Weight: 100, - SpreadTarget: []*structs.SpreadTarget{ - { - Value: "dc1", - Percent: start, - }, - { - Value: "dc2", - Percent: remaining, - }, - }, }) assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") // Create some nodes, half in dc2 @@ -692,1117 +539,1094 @@ func TestServiceSched_Spread(t *testing.T) { } assert.Len(planned, 10, "Planned Allocations") + // Expect even split allocs across datacenter expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 10 - i - if i > 0 { - expectedCounts["dc2"] = i - } + expectedCounts["dc1"] = 5 + expectedCounts["dc2"] = 5 + require.Equal(t, expectedCounts, dcAllocsMap) h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - start = start - step - } -} - -// Test job registration with even spread across dc -func TestServiceSched_EvenSpread(t *testing.T) { - assert := assert.New(t) - - h := NewHarness(t) - // Create a job that uses even spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c } - assert.Len(planned, 10, "Planned Allocations") - - // Expect even split allocs across datacenter - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 5 - expectedCounts["dc2"] = 5 - - require.Equal(t, expectedCounts, dcAllocsMap) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - expected := &structs.DesiredUpdates{Place: 10} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + expected := &structs.DesiredUpdates{Place: 10} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + } + }) } } func TestServiceSched_JobRegister_CountZero(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create NO nodes - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create NO nodes + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { - t.Fatalf("bad: %#v", metrics) - } + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { + t.Fatalf("bad: %#v", metrics) + } - // Check queued allocations - queued := outEval.QueuedAllocations["web"] - if queued != 10 { - t.Fatalf("expected queued: %v, actual: %v", 10, queued) + // Check queued allocations + queued := outEval.QueuedAllocations["web"] + if queued != 10 { + t.Fatalf("expected queued: %v, actual: %v", 10, queued) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a full node + node := mock.Node() + node.ReservedResources = &structs.NodeReservedResources{ + Cpu: structs.NodeReservedCpuResources{ + CpuShares: node.NodeResources.Cpu.CpuShares, + }, + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a full node - node := mock.Node() - node.ReservedResources = &structs.NodeReservedResources{ - Cpu: structs.NodeReservedCpuResources{ - CpuShares: node.NodeResources.Cpu.CpuShares, - }, - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create an ineligible node + node2 := mock.Node() + node2.Attributes["kernel.name"] = "windows" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create an ineligible node - node2 := mock.Node() - node2.Attributes["kernel.name"] = "windows" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + // Create a jobs + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a jobs - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan has created a follow up eval. + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan has created a follow up eval. - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + created := h.CreateEvals[0] + if created.Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", created) + } - created := h.CreateEvals[0] - if created.Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", created) - } + classes := created.ClassEligibility + if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { + t.Fatalf("bad: %#v", classes) + } - classes := created.ClassEligibility - if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { - t.Fatalf("bad: %#v", classes) - } + if created.EscapedComputedClass { + t.Fatalf("bad: %#v", created) + } - if created.EscapedComputedClass { - t.Fatalf("bad: %#v", created) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { + t.Fatalf("bad: %#v", metrics) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { - t.Fatalf("bad: %#v", metrics) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create one node - node := mock.Node() - node.NodeClass = "class_0" - noErr(t, node.ComputeClass()) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create one node + node := mock.Node() + node.NodeClass = "class_0" + noErr(t, node.ComputeClass()) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job that constrains on a node class - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].Constraints = append(job.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "class_0", - Operand: "=", - }, - ) - tg2 := job.TaskGroups[0].Copy() - tg2.Name = "web2" - tg2.Constraints[1].RTarget = "class_1" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job that constrains on a node class + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].Constraints = append(job.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "class_0", + Operand: "=", + }, + ) + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "web2" + tg2.Constraints[1].RTarget = "class_1" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 2 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 2 { + t.Fatalf("bad: %#v", plan) + } - // Ensure two allocations placed - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Ensure two allocations placed + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan failed to alloc one tg - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan failed to alloc one tg + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - metrics, ok := outEval.FailedTGAllocs[tg2.Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + metrics, ok := outEval.FailedTGAllocs[tg2.Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Check the coalesced failures - if metrics.CoalescedFailures != tg2.Count-1 { - t.Fatalf("bad: %#v", metrics) - } + // Check the coalesced failures + if metrics.CoalescedFailures != tg2.Count-1 { + t.Fatalf("bad: %#v", metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This test just ensures the scheduler handles the eval type to avoid // regressions. func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerMaxPlans, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerMaxPlans, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_Plan_Partial_Progress(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job with a high resource ask so that all the allocations can't - // be placed on a single node. - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job with a high resource ask so that all the allocations can't + // be placed on a single node. + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure only one allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure only one allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_EvaluateBlockedEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure that the eval was reblocked - if len(h.ReblockEvals) != 1 { - t.Fatalf("bad: %#v", h.ReblockEvals) - } - if h.ReblockEvals[0].ID != eval.ID { - t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) - } + // Ensure that the eval was reblocked + if len(h.ReblockEvals) != 1 { + t.Fatalf("bad: %#v", h.ReblockEvals) + } + if h.ReblockEvals[0].ID != eval.ID { + t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) + } - // Ensure the eval status was not updated - if len(h.Evals) != 0 { - t.Fatalf("Existing eval should not have status set") + // Ensure the eval status was not updated + if len(h.Evals) != 0 { + t.Fatalf("Existing eval should not have status set") + } + }) } } func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job and set the task group count to zero. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the eval has no spawned blocked eval - if len(h.Evals) != 1 { - t.Fatalf("bad: %#v", h.Evals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure the eval has no spawned blocked eval + if len(h.Evals) != 1 { + t.Fatalf("bad: %#v", h.Evals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure the eval was not reblocked - if len(h.ReblockEvals) != 0 { - t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") - } + // Ensure the eval was not reblocked + if len(h.ReblockEvals) != 0 { + t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure queued allocations is zero - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued: %v, actual: %v", 0, queued) + // Ensure queued allocations is zero + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued: %v, actual: %v", 0, queued) + } + }) } } func TestServiceSched_JobModify(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID + // Update the job + job2 := mock.Job() + job2.ID = job.ID - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // Have a single node and submit a job. Increment the count such that all fit // on the node but the node doesn't have enough resources to fit the new count + // 1. This tests that we properly discount the resources of existing allocs. func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { - h := NewHarness(t) - - // Create one node - node := mock.Node() - node.NodeResources.Cpu.CpuShares = 1000 - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with one allocation - job := mock.Job() - job.TaskGroups[0].Tasks[0].Resources.CPU = 256 - job2 := job.Copy() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Create one node + node := mock.Node() + node.NodeResources.Cpu.CpuShares = 1000 + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Update the job to count 3 - job2.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Generate a fake job with one allocation + job := mock.Job() + job.TaskGroups[0].Tasks[0].Resources.CPU = 256 + job2 := job.Copy() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + var allocs []*structs.Allocation + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job to count 3 + job2.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan didn't evicted the alloc - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't evicted the alloc + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan had no failures - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - if outEval == nil || len(outEval.FailedTGAllocs) != 0 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan had no failures + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + if outEval == nil || len(outEval.FailedTGAllocs) != 0 { + t.Fatalf("bad: %#v", outEval) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_JobModify_CountZero(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - // Update the job to be count zero - job2 := mock.Job() - job2.ID = job.ID - job2.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job to be count zero + job2 := mock.Job() + job2.ID = job.ID + job2.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != desiredUpdates { - t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != desiredUpdates { + t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) } } @@ -1811,2206 +1635,2578 @@ func TestServiceSched_JobModify_Rolling(t *testing.T) { // allocations as this allows us to assert that destructive changes are done // first. func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a node and clear the reserved resources + node := mock.Node() + node.ReservedResources = nil + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a resource ask that is the same as the resources available on the + // node + cpu := node.NodeResources.Cpu.CpuShares + mem := node.NodeResources.Memory.MemoryMB + + request := &structs.Resources{ + CPU: int(cpu), + MemoryMB: int(mem), + } + allocated := &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: cpu, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: mem, + }, + }, + }, + } - // Create a node and clear the reserved resources - node := mock.Node() - node.ReservedResources = nil - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Generate a fake job with one alloc that consumes the whole node + job := mock.Job() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Resources = request + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.AllocatedResources = allocated + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Update the job to place more versions of the task group, drop the count + // and force destructive updates + job2 := job.Copy() + job2.TaskGroups[0].Count = 5 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources - // Create a resource ask that is the same as the resources available on the - // node - cpu := node.NodeResources.Cpu.CpuShares - mem := node.NodeResources.Memory.MemoryMB + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - request := &structs.Resources{ - CPU: int(cpu), - MemoryMB: int(mem), - } - allocated := &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: cpu, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: mem, - }, - }, - }, - } + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Generate a fake job with one alloc that consumes the whole node - job := mock.Job() - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Tasks[0].Resources = request - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - alloc := mock.Alloc() - alloc.AllocatedResources = allocated - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Update the job to place more versions of the task group, drop the count - // and force destructive updates - job2 := job.Copy() - job2.TaskGroups[0].Count = 5 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 1 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) + } - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 1 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 5 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 5 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) } } func TestServiceSched_JobModify_Canaries(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 2 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - Canary: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 2 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + Canary: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted nothing - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) - } + // Ensure the plan evicted nothing + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - for _, canary := range planned { - if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { - t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) - } - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { - t.Fatalf("bad: %#v", state) - } + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { + t.Fatalf("bad: %#v", state) + } - // Assert the canaries were added to the placed list - if len(state.PlacedCanaries) != desiredUpdates { - t.Fatalf("bad: %#v", state) + // Assert the canaries were added to the placed list + if len(state.PlacedCanaries) != desiredUpdates { + t.Fatalf("bad: %#v", state) + } + }) } } func TestServiceSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations and create an older deployment - job := mock.Job() - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create allocs that are part of the old deployment - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = d.ID - alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations and create an older deployment + job := mock.Job() + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Create allocs that are part of the old deployment + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } } - } - } - // Verify the deployment id was changed and health cleared - for _, alloc := range out { - if alloc.DeploymentID == d.ID { - t.Fatalf("bad: deployment id not cleared") - } else if alloc.DeploymentStatus != nil { - t.Fatalf("bad: deployment status not cleared") - } + // Verify the deployment id was changed and health cleared + for _, alloc := range out { + if alloc.DeploymentID == d.ID { + t.Fatalf("bad: deployment id not cleared") + } else if alloc.DeploymentStatus != nil { + t.Fatalf("bad: deployment status not cleared") + } + } + }) } } func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - node.Meta["rack"] = fmt.Sprintf("rack%d", i) - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + node.Meta["rack"] = fmt.Sprintf("rack%d", i) + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - oldJob := job.Copy() - oldJob.JobModifyIndex -= 1 - oldJob.TaskGroups[0].Count = 4 + oldJob := job.Copy() + oldJob.JobModifyIndex -= 1 + oldJob.TaskGroups[0].Count = 4 - // Place 4 of 10 - var allocs []*structs.Allocation - for i := 0; i < 4; i++ { - alloc := mock.Alloc() - alloc.Job = oldJob - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Place 4 of 10 + var allocs []*structs.Allocation + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = oldJob + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", planned) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", planned) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations - job := mock.Job() + // Generate a fake job with allocations + job := mock.Job() - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all nodes - if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all nodes + if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - if alloc.Job == nil { - t.Fatalf("bad: %#v", alloc) - } - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + if alloc.Job == nil { + t.Fatalf("bad: %#v", alloc) + } + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) - require := require.New(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) - // Generate a fake job with allocations - job := mock.Job() - job.Stop = true - require.NoError(h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Stop = true + require.NoError(h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a summary where the queued allocs are set as we want to assert - // they get zeroed out. - summary := mock.JobSummary(job.ID) - web := summary.Summary["web"] - web.Queued = 2 - require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) + // Create a summary where the queued allocs are set as we want to assert + // they get zeroed out. + summary := mock.JobSummary(job.ID) + web := summary.Summary["web"] + web.Queued = 2 + require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - require.NoError(h.Process(NewServiceScheduler, eval)) + // Process the evaluation + require.NoError(h.Process(NewServiceScheduler, eval)) - // Ensure a single plan - require.Len(h.Plans, 1) - plan := h.Plans[0] + // Ensure a single plan + require.Len(h.Plans, 1) + plan := h.Plans[0] - // Ensure the plan evicted all nodes - require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) + // Ensure the plan evicted all nodes + require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(err) - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - require.NotNil(alloc.Job) - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + require.NotNil(alloc.Job) + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - require.Empty(out) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + require.Empty(out) - // Assert the job summary is cleared out - sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) - require.NoError(err) - require.NotNil(sout) - require.Contains(sout.Summary, "web") - webOut := sout.Summary["web"] - require.Zero(webOut.Queued) + // Assert the job summary is cleared out + sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) + require.NoError(err) + require.NotNil(sout) + require.Contains(sout.Summary, "web") + webOut := sout.Summary["web"] + require.Zero(webOut.Queued) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_NodeDown(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } - // Cover each terminal case and ensure it doesn't change to lost - allocs[7].DesiredStatus = structs.AllocDesiredStatusRun - allocs[7].ClientStatus = structs.AllocClientStatusLost - allocs[8].DesiredStatus = structs.AllocDesiredStatusRun - allocs[8].ClientStatus = structs.AllocClientStatusFailed - allocs[9].DesiredStatus = structs.AllocDesiredStatusRun - allocs[9].ClientStatus = structs.AllocClientStatusComplete - - // Mark some allocs as running - for i := 0; i < 4; i++ { - out := allocs[i] - out.ClientStatus = structs.AllocClientStatusRunning - } + // Cover each terminal case and ensure it doesn't change to lost + allocs[7].DesiredStatus = structs.AllocDesiredStatusRun + allocs[7].ClientStatus = structs.AllocClientStatusLost + allocs[8].DesiredStatus = structs.AllocDesiredStatusRun + allocs[8].ClientStatus = structs.AllocClientStatusFailed + allocs[9].DesiredStatus = structs.AllocDesiredStatusRun + allocs[9].ClientStatus = structs.AllocClientStatusComplete + + // Mark some allocs as running + for i := 0; i < 4; i++ { + out := allocs[i] + out.ClientStatus = structs.AllocClientStatusRunning + } - // Mark appropriate allocs for migration - for i := 0; i < 7; i++ { - out := allocs[i] - out.DesiredTransition.Migrate = helper.BoolToPtr(true) - } + // Mark appropriate allocs for migration + for i := 0; i < 7; i++ { + out := allocs[i] + out.DesiredTransition.Migrate = helper.BoolToPtr(true) + } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Test the scheduler marked all non-terminal allocations as lost - if len(plan.NodeUpdate[node.ID]) != 7 { - t.Fatalf("bad: %#v", plan) - } + // Test the scheduler marked all non-terminal allocations as lost + if len(plan.NodeUpdate[node.ID]) != 7 { + t.Fatalf("bad: %#v", plan) + } - for _, out := range plan.NodeUpdate[node.ID] { - if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad alloc: %#v", out) - } - } + for _, out := range plan.NodeUpdate[node.ID] { + if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad alloc: %#v", out) + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Mark some allocs as running + ws := memdb.NewWatchSet() + for i := 0; i < 4; i++ { + out, _ := h.State.AllocByID(ws, allocs[i].ID) + out.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) + } - // Mark some allocs as running - ws := memdb.NewWatchSet() - for i := 0; i < 4; i++ { - out, _ := h.State.AllocByID(ws, allocs[i].ID) - out.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) - } + // Create a mock evaluation which won't trigger any new placements + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation which won't trigger any new placements - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDrain(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestServiceSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Set the desired state of the allocs to stop + var stop []*structs.Allocation + for i := 0; i < 6; i++ { + newAlloc := allocs[i].Copy() + newAlloc.ClientStatus = structs.AllocDesiredStatusStop + newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + stop = append(stop, newAlloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) + + // Mark some of the allocations as running + var running []*structs.Allocation + for i := 4; i < 6; i++ { + newAlloc := stop[i].Copy() + newAlloc.ClientStatus = structs.AllocClientStatusRunning + running = append(running, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) + + // Mark some of the allocations as complete + var complete []*structs.Allocation + for i := 6; i < 10; i++ { + newAlloc := allocs[i].Copy() + newAlloc.TaskStates = make(map[string]*structs.TaskState) + newAlloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + newAlloc.ClientStatus = structs.AllocClientStatusComplete + complete = append(complete, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - // Set the desired state of the allocs to stop - var stop []*structs.Allocation - for i := 0; i < 6; i++ { - newAlloc := allocs[i].Copy() - newAlloc.ClientStatus = structs.AllocDesiredStatusStop - newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - stop = append(stop, newAlloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) - - // Mark some of the allocations as running - var running []*structs.Allocation - for i := 4; i < 6; i++ { - newAlloc := stop[i].Copy() - newAlloc.ClientStatus = structs.AllocClientStatusRunning - running = append(running, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) - - // Mark some of the allocations as complete - var complete []*structs.Allocation - for i := 6; i < 10; i++ { - newAlloc := allocs[i].Copy() - newAlloc.TaskStates = make(map[string]*structs.TaskState) - newAlloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - newAlloc.ClientStatus = structs.AllocClientStatusComplete - complete = append(complete, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 6 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 6 { - t.Fatalf("bad: %#v", plan) - } + // Ensure that all the allocations which were in running or pending state + // has been marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + sort.Strings(lostAllocs) - // Ensure that all the allocations which were in running or pending state - // has been marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - sort.Strings(lostAllocs) + var expectedLostAllocs []string + for i := 0; i < 6; i++ { + expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) + } + sort.Strings(expectedLostAllocs) - var expectedLostAllocs []string - for i := 0; i < 6; i++ { - expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) - } - sort.Strings(expectedLostAllocs) + if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { + t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) + } - if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { - t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } + }) } } func TestServiceSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) + }) + } } func TestServiceSched_Reschedule_OnceNow(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: 5 * time.Second, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: 5 * time.Second, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) + + // Mark this alloc as failed again, should not get rescheduled + newAlloc.ClientStatus = structs.AllocClientStatusFailed + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err = h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created this time + out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(3, len(out)) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +} - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) +// Tests that alloc reschedulable at a future time creates a follow up eval +func TestServiceSched_Reschedule_Later(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + delayDuration := 15 * time.Second + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: delayDuration, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now}} + failedAllocID := allocs[1].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify no new allocs were created + require.Equal(2, len(out)) + + // Verify follow up eval was created for the failed alloc + alloc, err := h.State.AllocByID(ws, failedAllocID) + require.Nil(err) + require.NotEmpty(alloc.FollowupEvalID) + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { + t.Fatalf("bad: %#v", h.CreateEvals) + } + followupEval := h.CreateEvals[0] + require.Equal(now.Add(delayDuration), followupEval.WaitUntil) + }) } +} + +func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + maxRestartAttempts := 3 + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: maxRestartAttempts, + Interval: 30 * time.Minute, + Delay: 5 * time.Second, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + expectedNumAllocs := 3 + expectedNumReschedTrackers := 1 - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) + failedAllocId := allocs[1].ID + failedNodeID := allocs[1].NodeID - // Mark this alloc as failed again, should not get rescheduled - newAlloc.ClientStatus = structs.AllocClientStatusFailed + assert := assert.New(t) + for i := 0; i < maxRestartAttempts; i++ { + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + noErr(t, err) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that a new allocation got created with its restart tracker info + assert.Equal(expectedNumAllocs, len(out)) + + // Find the new alloc with ClientStatusPending + var pendingAllocs []*structs.Allocation + var prevFailedAlloc *structs.Allocation + + for _, alloc := range out { + if alloc.ClientStatus == structs.AllocClientStatusPending { + pendingAllocs = append(pendingAllocs, alloc) + } + if alloc.ID == failedAllocId { + prevFailedAlloc = alloc + } + } + assert.Equal(1, len(pendingAllocs)) + newAlloc := pendingAllocs[0] + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) + + // Verify the previous NodeID in the most recent reschedule event + reschedEvents := newAlloc.RescheduleTracker.Events + assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) + assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) + + // Verify that the next alloc of the failed alloc is the newly rescheduled alloc + assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) + + // Mark this alloc as failed again + newAlloc.ClientStatus = structs.AllocClientStatusFailed + newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-12 * time.Second), + FinishedAt: now.Add(-10 * time.Second)}} + + failedAllocId = newAlloc.ID + failedNodeID = newAlloc.NodeID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + expectedNumAllocs += 1 + expectedNumReschedTrackers += 1 + } - // Process the evaluation - err = h.Process(NewServiceScheduler, eval) - assert.Nil(err) - // Verify no new allocs were created this time - out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(3, len(out)) + // Process last eval again, should not reschedule + err := h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created because restart attempts were exhausted + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts + }) + } } -// Tests that alloc reschedulable at a future time creates a follow up eval -func TestServiceSched_Reschedule_Later(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } +// Tests that old reschedule attempts are pruned +func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - delayDuration := 15 * time.Second - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: delayDuration, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + DelayFunction: "exponential", + MaxDelay: 1 * time.Hour, + Delay: 5 * time.Second, + Unlimited: true, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now}} - failedAllocID := allocs[1].ID + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + now := time.Now() + // Mark allocations as failed with restart info + allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-15 * time.Minute)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + allocs[1].RescheduleTracker = &structs.RescheduleTracker{ + Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + Delay: 5 * time.Second, + }, + {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 10 * time.Second, + }, + {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 20 * time.Second, + }, + {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 40 * time.Second, + }, + {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 80 * time.Second, + }, + {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 160 * time.Second, + }, + }, + } + expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] + expectedDelay := 320 * time.Second + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Verify no new allocs were created - require.Equal(2, len(out)) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } - // Verify follow up eval was created for the failed alloc - alloc, err := h.State.AllocByID(ws, failedAllocID) - require.Nil(err) - require.NotEmpty(alloc.FollowupEvalID) + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + // Verify that the new alloc copied the last 5 reschedule attempts + assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { - t.Fatalf("bad: %#v", h.CreateEvals) + mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] + // Verify that the failed alloc ID is in the most recent reschedule event + assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) + // Verify that the delay value was captured correctly + assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) + }) } - followupEval := h.CreateEvals[0] - require.Equal(now.Add(delayDuration), followupEval.WaitUntil) } -func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and a reschedule policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + } + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) + + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed + } + + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = deployment.ID + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + }) + } + }) } +} - maxRestartAttempts := 3 - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: maxRestartAttempts, - Interval: 30 * time.Minute, - Delay: 5 * time.Second, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() +func TestBatchSched_Run_CompleteAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.ClientStatus = structs.AllocClientStatusRunning - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - expectedNumAllocs := 3 - expectedNumReschedTrackers := 1 + // Ensure no plan as it should be a no-op + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - failedAllocId := allocs[1].ID - failedNodeID := allocs[1].NodeID + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - assert := assert.New(t) - for i := 0; i < maxRestartAttempts; i++ { - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - noErr(t, err) + // Ensure no allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) +func TestBatchSched_Run_FailedAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Verify that a new allocation got created with its restart tracker info - assert.Equal(expectedNumAllocs, len(out)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Find the new alloc with ClientStatusPending - var pendingAllocs []*structs.Allocation - var prevFailedAlloc *structs.Allocation + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - for _, alloc := range out { - if alloc.ClientStatus == structs.AllocClientStatusPending { - pendingAllocs = append(pendingAllocs, alloc) - } - if alloc.ID == failedAllocId { - prevFailedAlloc = alloc + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, } - } - assert.Equal(1, len(pendingAllocs)) - newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) - - // Verify the previous NodeID in the most recent reschedule event - reschedEvents := newAlloc.RescheduleTracker.Events - assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) - assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) - - // Verify that the next alloc of the failed alloc is the newly rescheduled alloc - assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) - - // Mark this alloc as failed again - newAlloc.ClientStatus = structs.AllocClientStatusFailed - newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-12 * time.Second), - FinishedAt: now.Add(-10 * time.Second)}} - - failedAllocId = newAlloc.ID - failedNodeID = newAlloc.NodeID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expectedNumAllocs += 1 - expectedNumReschedTrackers += 1 - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process last eval again, should not reschedule - err := h.Process(NewServiceScheduler, eval) - assert.Nil(err) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Verify no new allocs were created because restart attempts were exhausted - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts -} + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } -// Tests that old reschedule attempts are pruned -func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { - h := NewHarness(t) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure a replacement alloc was placed. + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - DelayFunction: "exponential", - MaxDelay: 1 * time.Hour, - Delay: 5 * time.Second, - Unlimited: true, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - now := time.Now() - // Mark allocations as failed with restart info - allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-15 * time.Minute)}} - allocs[1].ClientStatus = structs.AllocClientStatusFailed - - allocs[1].RescheduleTracker = &structs.RescheduleTracker{ - Events: []*structs.RescheduleEvent{ - {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), - PrevAllocID: uuid.Generate(), - PrevNodeID: uuid.Generate(), - Delay: 5 * time.Second, - }, - {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 10 * time.Second, - }, - {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 20 * time.Second, - }, - {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 40 * time.Second, - }, - {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 80 * time.Second, - }, - {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 160 * time.Second, - }, - }, + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] - expectedDelay := 320 * time.Second - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) +} - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +func TestBatchSched_Run_LostAlloc(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a job + job := mock.Job() + job.ID = "my-job" + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } + // Desired = 3 + // Mark one as lost and then schedule + // [(0, run, running), (1, run, running), (1, stop, lost)] - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - // Verify that the new alloc copied the last 5 reschedule attempts - assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) + // Create two running allocations + var allocs []*structs.Allocation + for i := 0; i <= 1; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } - mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] - // Verify that the failed alloc ID is in the most recent reschedule event - assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) - // Verify that the delay value was captured correctly - assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[1]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) -} + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) -// Tests that deployments with failed allocs result in placements as long as the -// deployment is running. -func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { - for _, failedDeployment := range []bool{false, true} { - t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } - // Generate a fake job with allocations and a reschedule policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - if failedDeployment { - deployment.Status = structs.DeploymentStatusFailed + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 4 { + t.Fatalf("bad: %#v", out) } - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + // Assert that we have the correct number of each alloc name + expected := map[string]int{ + "my-job.web[0]": 1, + "my-job.web[1]": 2, + "my-job.web[2]": 1, + } + actual := make(map[string]int, 3) + for _, alloc := range out { + actual[alloc.Name] += 1 + } + require.Equal(t, actual, expected) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID - allocs = append(allocs, alloc) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobRegister(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Mark one of the allocations as failed in the past - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", - StartedAt: time.Now().Add(-12 * time.Hour), - FinishedAt: time.Now().Add(-10 * time.Hour)}} - allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation + // Create a mock evaluation to register the job eval := &structs.Evaluation{ Namespace: structs.DefaultNamespace, ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, JobID: job.ID, Status: structs.EvalStatusPending, } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) - - if failedDeployment { - // Verify no plan created - require.Len(h.Plans, 0) - } else { - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } - }) - } -} - -func TestBatchSched_Run_CompleteAlloc(t *testing.T) { - h := NewHarness(t) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Ensure the eval has no spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no plan as it should be a no-op - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure different ports were used. + used := make(map[int]map[string]struct{}) + for _, alloc := range out { + for _, resource := range alloc.TaskResources { + for _, port := range resource.Networks[0].DynamicPorts { + nodeMap, ok := used[port.Value] + if !ok { + nodeMap = make(map[string]struct{}) + used[port.Value] = nodeMap + } + if _, ok := nodeMap[alloc.NodeID]; ok { + t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) + } + nodeMap[alloc.NodeID] = struct{}{} + } + } + } - // Ensure no allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestBatchSched_Run_FailedAlloc(t *testing.T) { - h := NewHarness(t) +func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - tgName := job.TaskGroups[0].Name - now := time.Now() + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan allocated + plan := h.Plans[0] + planned := make(map[string]*structs.Allocation) + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + planned[alloc.ID] = alloc + } + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the job to force a rolling upgrade + updated := job.Copy() + updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 + noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure we have created only one new allocation + // Ensure a single plan + if len(h1.Plans) != 1 { + t.Fatalf("bad: %#v", h1.Plans) + } + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 10 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocations were placed on the same node as the older + // ones + for _, new := range newPlanned { + if new.PreviousAllocation == "" { + t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) + } - // Ensure a replacement alloc was placed. - if len(out) != 2 { - t.Fatalf("bad: %#v", out) + old, ok := planned[new.PreviousAllocation] + if !ok { + t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) + } + if new.NodeID != old.NodeID { + t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) + } + } + }) } +} - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } +func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) -func TestBatchSched_Run_LostAlloc(t *testing.T) { - h := NewHarness(t) + // Create a job with count 2 and disk as 60GB so that only one allocation + // can fit + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + ID: uuid.Generate(), + } - // Create a job - job := mock.Job() - job.ID = "my-job" - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Desired = 3 - // Mark one as lost and then schedule - // [(0, run, running), (1, run, running), (1, stop, lost)] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create two running allocations - var allocs []*structs.Allocation - for i := 0; i <= 1; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[1]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the eval has a blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan allocated only one allocation + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure a replacement alloc was placed. - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Ensure only one allocation was placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Assert that we have the correct number of each alloc name - expected := map[string]int{ - "my-job.web[0]": 1, - "my-job.web[1]": 2, - "my-job.web[2]": 1, - } - actual := make(map[string]int, 3) - for _, alloc := range out { - actual[alloc.Name] += 1 + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - require.Equal(t, actual, expected) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { - h := NewHarness(t) - - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - tgName := job.TaskGroups[0].Name - now := time.Now() + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 1 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 1 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a successful alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a successful alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, }, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to rerun the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to rerun the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no replacement alloc was placed. - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure no replacement alloc was placed. + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // This test checks that terminal allocations that receive an in-place updated // are not added to the plan func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to trigger the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to trigger the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans[0]) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans[0]) + } + }) } } // This test ensures that terminal jobs from older versions are ignored. func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - job2.Type = structs.JobTypeBatch - job2.Version++ - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job + job2 := mock.Job() + job2.ID = job.ID + job2.Type = structs.JobTypeBatch + job2.Version++ + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - allocs = nil - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job2 - alloc.JobID = job2.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, + allocs = nil + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job2 + alloc.JobID = job2.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, }, - }, - }, - } - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + } + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) + // Ensure a plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + }) } } // This test asserts that an allocation from an old job that is running on a // drained node is cleaned up. func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a running alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create an update job + job2 := job.Copy() + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + job2.Version++ + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a running alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create an update job - job2 := job.Copy() - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - job2.Version++ - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + plan := h.Plans[0] - plan := h.Plans[0] + // Ensure the plan evicted 1 + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted 1 - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan places 1 + if len(plan.NodeAllocation[node2.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan places 1 - if len(plan.NodeAllocation[node2.ID]) != 1 { - t.Fatalf("bad: %#v", plan) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test asserts that an allocation from a job that is complete on a // drained node is ignored up. func TestBatchSched_NodeDrain_Complete(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = make(map[string]*structs.TaskState) + alloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = make(map[string]*structs.TaskState) - alloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This is a slightly odd test but it ensures that we handle a scale down of a @@ -4099,303 +4295,318 @@ func TestBatchSched_ScaleDown_SameName(t *testing.T) { } func TestGenericSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.Job() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - job1.TaskGroups[0].Count = 12 - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - if err := h1.Process(NewServiceScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.Job() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + job1.TaskGroups[0].Count = 12 + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - plan := h1.Plans[0] + // Process the evaluation + if err := h1.Process(NewServiceScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestServiceSched_NodeDrain_Sticky(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create an alloc on the draining node + alloc := mock.Alloc() + alloc.Name = "my-job.web[0]" + alloc.NodeID = node.ID + alloc.Job.TaskGroups[0].Count = 1 + alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: alloc.Job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Create an alloc on the draining node - alloc := mock.Alloc() - alloc.Name = "my-job.web[0]" - alloc.NodeID = node.ID - alloc.Job.TaskGroups[0].Count = 1 - alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: alloc.Job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't create any new allocations + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't create any new allocations - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is stopped, the scheduler properly cancels // an outstanding deployment. func TestServiceSched_CancelDeployment_Stopped(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job - job := mock.Job() - job.JobModifyIndex = job.CreateIndex + 1 - job.ModifyIndex = job.CreateIndex + 1 - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment - d := mock.Deployment() - d.JobID = job.ID - d.JobCreateIndex = job.CreateIndex - d.JobModifyIndex = job.JobModifyIndex - 1 - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Generate a fake job + job := mock.Job() + job.JobModifyIndex = job.CreateIndex + 1 + job.ModifyIndex = job.CreateIndex + 1 + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a deployment + d := mock.Deployment() + d.JobID = job.ID + d.JobCreateIndex = job.CreateIndex + d.JobModifyIndex = job.JobModifyIndex - 1 + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) + } - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) - } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is updated and had an old deployment, the scheduler properly cancels // the deployment. func TestServiceSched_CancelDeployment_NewerJob(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a deployment for an old version of the job - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Create a deployment for an old version of the job + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Upsert again to bump job version - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Upsert again to bump job version + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to kick the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to kick the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) - } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) + } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } // Various table driven tests for carry forward // of past reschedule events func Test_updateRescheduleTracker(t *testing.T) { - t1 := time.Now().UTC() alloc := mock.Alloc() prevAlloc := mock.Alloc() @@ -4626,5 +4837,4 @@ func Test_updateRescheduleTracker(t *testing.T) { require.Equal(tc.expectedRescheduleEvents, alloc.RescheduleTracker.Events) }) } - } diff --git a/scheduler/system_sched_test.go b/scheduler/system_sched_test.go index 9461fd85eaa..2509697de93 100644 --- a/scheduler/system_sched_test.go +++ b/scheduler/system_sched_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -15,1870 +15,1964 @@ import ( "github.com/stretchr/testify/require" ) +// COMPAT 0.11: Currently, all the tests run for 2 cases: +// 1) Allow plan optimization +// 2) Not allowing plan optimization +// The code for not allowing plan optimizations is in place to allow for safer cluster upgrades, +// and backwards compatibility with the existing raft logs. The older code will be removed later, +// and these tests should then no longer include the testing for case (2) + func TestSystemSched_JobRegister(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure no allocations are queued + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) + } - // Ensure no allocations are queued - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - plan := h.Plans[0] - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + plan := h.Plans[0] + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Get an allocation and mark it as failed - alloc := planned[4].Copy() - alloc.ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Get an allocation and mark it as failed + alloc := planned[4].Copy() + alloc.ClientStatus = structs.AllocClientStatusFailed + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure we have created only one new allocation - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 1 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocation was placed on the same node as the older - // one - if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { - t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + // Ensure we have created only one new allocation + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 1 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocation was placed on the same node as the older + // one + if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { + t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + } + }) } } func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { - h := NewHarness(t) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create another job with a lot of disk resource ask so that it doesn't fit - // the node - job1 := mock.SystemJob() - job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create another job with a lot of disk resource ask so that it doesn't fit + // the node + job1 := mock.SystemJob() + job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Create a new harness to test the scheduling result for the second job - h1 := NewHarnessWithState(t, h.State) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a new harness to test the scheduling result for the second job + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) - noErr(t, err) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) + out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) + noErr(t, err) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + }) } } func TestSystemSched_ExhaustResources(t *testing.T) { - h := NewHarness(t) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create a service job which consumes most of the system resources - svcJob := mock.Job() - svcJob.TaskGroups[0].Count = 1 - svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: svcJob.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: svcJob.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create a service job which consumes most of the system resources + svcJob := mock.Job() + svcJob.TaskGroups[0].Count = 1 + svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: svcJob.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: svcJob.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a system job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Create a system job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // System scheduler will preempt the service job and would have placed eval1 - require := require.New(t) + // System scheduler will preempt the service job and would have placed eval1 + require := require.New(t) - newPlan := h.Plans[1] - require.Len(newPlan.NodeAllocation, 1) - require.Len(newPlan.NodePreemptions, 1) + newPlan := h.Plans[1] + require.Len(newPlan.NodeAllocation, 1) + require.Len(newPlan.NodePreemptions, 1) - for _, allocList := range newPlan.NodeAllocation { - require.Len(allocList, 1) - require.Equal(job.ID, allocList[0].JobID) - } + for _, allocList := range newPlan.NodeAllocation { + require.Len(allocList, 1) + require.Equal(job.ID, allocList[0].JobID) + } - for _, allocList := range newPlan.NodePreemptions { - require.Len(allocList, 1) - require.Equal(svcJob.ID, allocList[0].JobID) - } - // Ensure that we have no queued allocations on the second eval - queued := h.Evals[1].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + for _, allocList := range newPlan.NodePreemptions { + require.Len(allocList, 1) + alloc, err := h.State.AllocByID(nil, allocList[0].ID) + noErr(t, err) + require.Equal(svcJob.ID, alloc.JobID) + } + // Ensure that we have no queued allocations on the second eval + queued := h.Evals[1].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestSystemSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - if i < 9 { - node.NodeClass = "foo" - } else { - node.NodeClass = "bar" - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + if i < 9 { + node.NodeClass = "foo" + } else { + node.NodeClass = "bar" + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job constraining on node class - job := mock.SystemJob() - fooConstraint := &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "foo", - Operand: "==", - } - job.Constraints = append(job.Constraints, fooConstraint) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job constraining on node class + job := mock.SystemJob() + fooConstraint := &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "foo", + Operand: "==", + } + job.Constraints = append(job.Constraints, fooConstraint) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 9 { - t.Fatalf("bad: %#v %d", planned, len(planned)) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 9 { + t.Fatalf("bad: %#v %d", planned, len(planned)) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 9 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 9 { + t.Fatalf("bad: %#v", out) + } - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - expected := &structs.DesiredUpdates{Place: 9} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + expected := &structs.DesiredUpdates{Place: 9} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + + } + }) } } func TestSystemSched_JobRegister_AddNode(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a new node. - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a new node. + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan had no node updates - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Log(len(update)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan had no node updates + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Log(len(update)) + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated on the new node - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated on the new node + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure it allocated on the right node - if _, ok := plan.NodeAllocation[node.ID]; !ok { - t.Fatalf("allocated on wrong node: %#v", plan) - } + // Ensure it allocated on the right node + if _, ok := plan.NodeAllocation[node.ID]; !ok { + t.Fatalf("allocated on wrong node: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 11 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 11 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) - - // Create NO nodes - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create NO nodes + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan as this should be a no-op. - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan as this should be a no-op. + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobModify(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = "my-job.web[0]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = "my-job.web[0]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - job2.Update = structs.UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 5, - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + job2.Update = structs.UpdateStrategy{ + Stagger: 30 * time.Second, + MaxParallel: 5, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a follow up eval was created - eval = h.Evals[0] - if eval.NextEval == "" { - t.Fatalf("missing next eval") - } + // Ensure a follow up eval was created + eval = h.Evals[0] + if eval.NextEval == "" { + t.Fatalf("missing next eval") + } - // Check for create - if len(h.CreateEvals) == 0 { - t.Fatalf("missing created eval") - } - create := h.CreateEvals[0] - if eval.NextEval != create.ID { - t.Fatalf("ID mismatch") - } - if create.PreviousEval != eval.ID { - t.Fatalf("missing previous eval") - } + // Check for create + if len(h.CreateEvals) == 0 { + t.Fatalf("missing created eval") + } + create := h.CreateEvals[0] + if eval.NextEval != create.ID { + t.Fatalf("ID mismatch") + } + if create.PreviousEval != eval.ID { + t.Fatalf("missing previous eval") + } - if create.TriggeredBy != structs.EvalTriggerRollingUpdate { - t.Fatalf("bad: %#v", create) + if create.TriggeredBy != structs.EvalTriggerRollingUpdate { + t.Fatalf("bad: %#v", create) + } + }) } } func TestSystemSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } + } + }) } } func TestSystemSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeDown(t *testing.T) { - h := NewHarness(t) - - // Register a down node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a down node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && - p.ClientStatus != structs.AllocClientStatusLost { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the allocations is stopped + if p := planned[0]; p.DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the allocation is marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - expected := []string{alloc.ID} + // Ensure that the allocation is marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + expected := []string{alloc.ID} - if !reflect.DeepEqual(lostAllocs, expected) { - t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) + if !reflect.DeepEqual(lostAllocs, expected) { + t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Log(len(planned)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Log(len(planned)) + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestSystemSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) + }) } - - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) } // This test ensures that the scheduler doesn't increment the queued allocation // count for a task group when allocations can't be created on currently // available nodes because of constrain mismatches. func TestSystemSched_Queued_With_Constraints(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - node.Attributes["kernel.name"] = "darwin" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a system job which can't be placed on the node - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + node.Attributes["kernel.name"] = "darwin" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a system job which can't be placed on the node + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } + }) } } func TestSystemSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.SystemJob() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Insert two more nodes - for i := 0; i < 2; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.SystemJob() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Insert two more nodes + for i := 0; i < 2; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - plan := h1.Plans[0] + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestSystemSched_PlanWithDrainedNode(t *testing.T) { - h := NewHarness(t) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.Drain = true - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.Drain = true + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create an allocation on each node + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + alloc.TaskGroup = "web" + + alloc2 := mock.Alloc() + alloc2.Job = job + alloc2.JobID = job.ID + alloc2.NodeID = node2.ID + alloc2.Name = "my-job.web2[0]" + alloc2.TaskGroup = "web2" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create an allocation on each node - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - alloc.TaskGroup = "web" - - alloc2 := mock.Alloc() - alloc2.Job = job - alloc2.JobID = job.ID - alloc2.NodeID = node2.ID - alloc2.Name = "my-job.web2[0]" - alloc2.TaskGroup = "web2" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan evicted the alloc on the failed node + planned := plan.NodeUpdate[node.ID] + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan evicted the alloc on the failed node - planned := plan.NodeUpdate[node.ID] - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan didn't place + if len(plan.NodeAllocation) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan didn't place - if len(plan.NodeAllocation) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { - h := NewHarness(t) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + qa := h.Evals[0].QueuedAllocations + if qa["web"] != 0 || qa["web2"] != 0 { + t.Fatalf("bad queued allocations %#v", qa) + } - qa := h.Evals[0].QueuedAllocations - if qa["web"] != 0 || qa["web2"] != 0 { - t.Fatalf("bad queued allocations %#v", qa) + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_Preemption(t *testing.T) { - h := NewHarness(t) - - // Create nodes - var nodes []*structs.Node - for i := 0; i < 2; i++ { - node := mock.Node() - //TODO(preetha): remove in 0.11 - node.Resources = &structs.Resources{ - CPU: 3072, - MemoryMB: 5034, - DiskMB: 20 * 1024, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - node.NodeResources = &structs.NodeResources{ - Cpu: structs.NodeCpuResources{ - CpuShares: 3072, - }, - Memory: structs.NodeMemoryResources{ - MemoryMB: 5034, - }, - Disk: structs.NodeDiskResources{ - DiskMB: 20 * 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - nodes = append(nodes, node) - } + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create nodes + var nodes []*structs.Node + for i := 0; i < 2; i++ { + node := mock.Node() + //TODO(preetha): remove in 0.11 + node.Resources = &structs.Resources{ + CPU: 3072, + MemoryMB: 5034, + DiskMB: 20 * 1024, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + node.NodeResources = &structs.NodeResources{ + Cpu: structs.NodeCpuResources{ + CpuShares: 3072, + }, + Memory: structs.NodeMemoryResources{ + MemoryMB: 5034, + }, + Disk: structs.NodeDiskResources{ + DiskMB: 20 * 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + nodes = append(nodes, node) + } - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create some low priority batch jobs and allocations for them - // One job uses a reserved port - job1 := mock.BatchJob() - job1.Type = structs.JobTypeBatch - job1.Priority = 20 - job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - ReservedPorts: []structs.Port{ + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create some low priority batch jobs and allocations for them + // One job uses a reserved port + job1 := mock.BatchJob() + job1.Type = structs.JobTypeBatch + job1.Priority = 20 + job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ { - Label: "web", - Value: 80, + MBits: 200, + ReservedPorts: []structs.Port{ + { + Label: "web", + Value: 80, + }, + }, }, }, - }, - }, - } + } - alloc1 := mock.Alloc() - alloc1.Job = job1 - alloc1.JobID = job1.ID - alloc1.NodeID = nodes[0].ID - alloc1.Name = "my-job[0]" - alloc1.TaskGroup = job1.TaskGroups[0].Name - alloc1.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, + alloc1 := mock.Alloc() + alloc1.Job = job1 + alloc1.JobID = job1.ID + alloc1.NodeID = nodes[0].ID + alloc1.Name = "my-job[0]" + alloc1.TaskGroup = job1.TaskGroups[0].Name + alloc1.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 200, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + job2 := mock.BatchJob() + job2.Type = structs.JobTypeBatch + job2.Priority = 20 + job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 200, + MBits: 200, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - job2 := mock.BatchJob() - job2.Type = structs.JobTypeBatch - job2.Priority = 20 - job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - }, - }, - } + } - alloc2 := mock.Alloc() - alloc2.Job = job2 - alloc2.JobID = job2.ID - alloc2.NodeID = nodes[0].ID - alloc2.Name = "my-job[2]" - alloc2.TaskGroup = job2.TaskGroups[0].Name - alloc2.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, + alloc2 := mock.Alloc() + alloc2.Job = job2 + alloc2.JobID = job2.ID + alloc2.NodeID = nodes[0].ID + alloc2.Name = "my-job[2]" + alloc2.TaskGroup = job2.TaskGroups[0].Name + alloc2.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + MBits: 200, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + job3 := mock.Job() + job3.Type = structs.JobTypeBatch + job3.Priority = 40 + job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, Networks: []*structs.NetworkResource{ { Device: "eth0", - IP: "192.168.0.100", - MBits: 200, + MBits: 400, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - job3 := mock.Job() - job3.Type = structs.JobTypeBatch - job3.Priority = 40 - job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - MBits: 400, - }, - }, - } + } - alloc3 := mock.Alloc() - alloc3.Job = job3 - alloc3.JobID = job3.ID - alloc3.NodeID = nodes[0].ID - alloc3.Name = "my-job[0]" - alloc3.TaskGroup = job3.TaskGroups[0].Name - alloc3.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, + alloc3 := mock.Alloc() + alloc3.Job = job3 + alloc3.JobID = job3.ID + alloc3.NodeID = nodes[0].ID + alloc3.Name = "my-job[0]" + alloc3.TaskGroup = job3.TaskGroups[0].Name + alloc3.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 25, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 400, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 25, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) + + // Create a high priority job and allocs for it + // These allocs should not be preempted + + job4 := mock.BatchJob() + job4.Type = structs.JobTypeBatch + job4.Priority = 100 + job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 400, + MBits: 100, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) - - // Create a high priority job and allocs for it - // These allocs should not be preempted - - job4 := mock.BatchJob() - job4.Type = structs.JobTypeBatch - job4.Priority = 100 - job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - MBits: 100, - }, - }, - } + } - alloc4 := mock.Alloc() - alloc4.Job = job4 - alloc4.JobID = job4.ID - alloc4.NodeID = nodes[0].ID - alloc4.Name = "my-job4[0]" - alloc4.TaskGroup = job4.TaskGroups[0].Name - alloc4.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, + alloc4 := mock.Alloc() + alloc4.Job = job4 + alloc4.JobID = job4.ID + alloc4.NodeID = nodes[0].ID + alloc4.Name = "my-job4[0]" + alloc4.TaskGroup = job4.TaskGroups[0].Name + alloc4.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 2048, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 100, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2048, + Shared: structs.AllocatedSharedResources{ + DiskMB: 2 * 1024, }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) + + // Create a system job such that it would need to preempt both allocs to succeed + job := mock.SystemJob() + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1948, + MemoryMB: 256, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 100, + MBits: 800, + DynamicPorts: []structs.Port{{Label: "http"}}, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 2 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) - - // Create a system job such that it would need to preempt both allocs to succeed - job := mock.SystemJob() - job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1948, - MemoryMB: 256, - Networks: []*structs.NetworkResource{ - { - MBits: 800, - DynamicPorts: []structs.Port{{Label: "http"}}, - }, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - require := require.New(t) - require.Nil(err) - - // Ensure a single plan - require.Equal(1, len(h.Plans)) - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - require.Nil(plan.Annotations) - - // Ensure the plan allocated on both nodes - var planned []*structs.Allocation - preemptingAllocId := "" - require.Equal(2, len(plan.NodeAllocation)) - - // The alloc that got placed on node 1 is the preemptor - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - for _, alloc := range allocList { - if alloc.NodeID == nodes[0].ID { - preemptingAllocId = alloc.ID } - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + require := require.New(t) + require.Nil(err) + + // Ensure a single plan + require.Equal(1, len(h.Plans)) + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + require.Nil(plan.Annotations) + + // Ensure the plan allocated on both nodes + var planned []*structs.Allocation + preemptingAllocId := "" + require.Equal(2, len(plan.NodeAllocation)) + + // The alloc that got placed on node 1 is the preemptor + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + for _, alloc := range allocList { + if alloc.NodeID == nodes[0].ID { + preemptingAllocId = alloc.ID + } + } + } - // Ensure all allocations placed - require.Equal(2, len(out)) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Verify that one node has preempted allocs - require.NotNil(plan.NodePreemptions[nodes[0].ID]) - preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + // Ensure all allocations placed + require.Equal(2, len(out)) - // Verify that three jobs have preempted allocs - require.Equal(3, len(preemptedAllocs)) + // Verify that one node has preempted allocs + require.NotNil(plan.NodePreemptions[nodes[0].ID]) + preemptedAllocs := plan.NodePreemptions[nodes[0].ID] - expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} + // Verify that three jobs have preempted allocs + require.Equal(3, len(preemptedAllocs)) - // We expect job1, job2 and job3 to have preempted allocations - // job4 should not have any allocs preempted - for _, alloc := range preemptedAllocs { - require.Contains(expectedPreemptedJobIDs, alloc.JobID) - } - // Look up the preempted allocs by job ID - ws = memdb.NewWatchSet() - - for _, jobId := range expectedPreemptedJobIDs { - out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) - noErr(t, err) - for _, alloc := range out { - require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) - require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) - } - } + expectedPreemptedAllocIDs := []string{alloc1.ID, alloc2.ID, alloc3.ID} - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // We expect job1, job2 and job3 to have preempted allocations + // job4 should not have any allocs preempted + for _, alloc := range preemptedAllocs { + require.Contains(expectedPreemptedAllocIDs, alloc.ID) + } + // Look up the preempted allocs by job ID + ws = memdb.NewWatchSet() + + for _, allocID := range expectedPreemptedAllocIDs { + evictedAlloc, err := h.State.AllocByID(ws, allocID) + noErr(t, err) + require.Equal(structs.AllocDesiredStatusEvict, evictedAlloc.DesiredStatus) + require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), evictedAlloc.DesiredDescription) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } diff --git a/scheduler/testing.go b/scheduler/testing.go index 61fa7f79c61..dfe4f728639 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -5,9 +5,9 @@ import ( "sync" "time" - testing "github.com/mitchellh/go-testing-interface" + "github.com/mitchellh/go-testing-interface" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" @@ -53,26 +53,30 @@ type Harness struct { nextIndex uint64 nextIndexLock sync.Mutex + + allowPlanOptimization bool } // NewHarness is used to make a new testing harness -func NewHarness(t testing.T) *Harness { +func NewHarness(t testing.T, allowPlanOptimization bool) *Harness { state := state.TestStateStore(t) h := &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } return h } // NewHarnessWithState creates a new harness with the given state for testing // purposes. -func NewHarnessWithState(t testing.T, state *state.StateStore) *Harness { +func NewHarnessWithState(t testing.T, state *state.StateStore, allowPlanOptimization bool) *Harness { return &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } } @@ -101,22 +105,17 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er result.AllocIndex = index // Flatten evicts and allocs - var allocs []*structs.Allocation + now := time.Now().UTC().UnixNano() + allocsStopped := make([]*structs.Allocation, 0, len(result.NodeUpdate)) for _, updateList := range plan.NodeUpdate { - allocs = append(allocs, updateList...) - } - for _, allocList := range plan.NodeAllocation { - allocs = append(allocs, allocList...) + allocsStopped = append(allocsStopped, updateList...) } - // Set the time the alloc was applied for the first time. This can be used - // to approximate the scheduling time. - now := time.Now().UTC().UnixNano() - for _, alloc := range allocs { - if alloc.CreateTime == 0 { - alloc.CreateTime = now - } + allocsUpdated := make([]*structs.Allocation, 0, len(result.NodeAllocation)) + for _, allocList := range plan.NodeAllocation { + allocsUpdated = append(allocsUpdated, allocList...) } + updateCreateTimestamp(allocsUpdated, now) // Set modify time for preempted allocs and flatten them var preemptedAllocs []*structs.Allocation @@ -130,8 +129,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: plan.Job, - Alloc: allocs, + Job: plan.Job, }, Deployment: plan.Deployment, DeploymentUpdates: plan.DeploymentUpdates, @@ -139,11 +137,33 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er NodePreemptions: preemptedAllocs, } + if h.allowPlanOptimization { + req.AllocsStopped = allocsStopped + req.AllocsUpdated = allocsUpdated + } else { + // Deprecated: Handles unoptimized log format + var allocs []*structs.Allocation + allocs = append(allocs, allocsStopped...) + allocs = append(allocs, allocsUpdated...) + updateCreateTimestamp(allocs, now) + req.Alloc = allocs + } + // Apply the full plan err := h.State.UpsertPlanResults(index, &req) return result, nil, err } +func updateCreateTimestamp(allocations []*structs.Allocation, now int64) { + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + for _, alloc := range allocations { + if alloc.CreateTime == 0 { + alloc.CreateTime = now + } + } +} + func (h *Harness) UpdateEval(eval *structs.Evaluation) error { // Ensure sequential plan application h.planLock.Lock() @@ -214,15 +234,15 @@ func (h *Harness) Snapshot() State { // Scheduler is used to return a new scheduler from // a snapshot of current state using the harness for planning. -func (h *Harness) Scheduler(factory Factory) Scheduler { +func (h *Harness) Scheduler(factory Factory, allowPlanOptimization bool) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h, false) + return factory(logger, h.Snapshot(), h, allowPlanOptimization) } // Process is used to process an evaluation given a factory // function to create the scheduler func (h *Harness) Process(factory Factory, eval *structs.Evaluation) error { - sched := h.Scheduler(factory) + sched := h.Scheduler(factory, h.allowPlanOptimization) return sched.Process(eval) } diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 08f5812aac6..13864b43214 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -621,7 +621,7 @@ func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { } func TestSetStatus(t *testing.T) { - h := NewHarness(t) + h := NewHarness(t, true) logger := testlog.HCLogger(t) eval := mock.Eval() status := "a" @@ -640,7 +640,7 @@ func TestSetStatus(t *testing.T) { } // Test next evals - h = NewHarness(t) + h = NewHarness(t, true) next := mock.Eval() if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -656,7 +656,7 @@ func TestSetStatus(t *testing.T) { } // Test blocked evals - h = NewHarness(t) + h = NewHarness(t, true) blocked := mock.Eval() if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -672,7 +672,7 @@ func TestSetStatus(t *testing.T) { } // Test metrics - h = NewHarness(t) + h = NewHarness(t, true) metrics := map[string]*structs.AllocMetric{"foo": nil} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -688,7 +688,7 @@ func TestSetStatus(t *testing.T) { } // Test queued allocations - h = NewHarness(t) + h = NewHarness(t, true) queuedAllocs := map[string]int{"web": 1} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { @@ -704,7 +704,7 @@ func TestSetStatus(t *testing.T) { t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) } - h = NewHarness(t) + h = NewHarness(t, true) dID := uuid.Generate() if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { t.Fatalf("setStatus() failed: %v", err) From 02b832c3ffe713bc9a8c30e675891dae2d25b0f0 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Fri, 8 Mar 2019 03:18:56 -0800 Subject: [PATCH 3/7] Compat tags --- nomad/fsm.go | 2 +- nomad/job_endpoint.go | 2 +- nomad/plan_apply.go | 40 ++++++++++++++++++++------------- nomad/plan_apply_test.go | 2 +- nomad/structs/structs.go | 1 + nomad/util.go | 6 ++--- scheduler/generic_sched_test.go | 2 +- scheduler/testing.go | 2 +- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/nomad/fsm.go b/nomad/fsm.go index fc3cebad1b4..c54a40d86da 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -1427,7 +1427,7 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanDenormalization, true) + allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanNormalization, true) sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) if err != nil { return err diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 332c048a4c7..8659929f9aa 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1216,7 +1216,7 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanDenormalization, true) + allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanNormalization, true) sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) if err != nil { return err diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index dc991323288..777b5e8cc6c 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -163,7 +163,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap preemptedJobIDs := make(map[structs.NamespacedID]struct{}) now := time.Now().UTC().UnixNano() - if ServersMeetMinimumVersion(p.Members(), MinVersionPlanDenormalization, true) { + if ServersMeetMinimumVersion(p.Members(), MinVersionPlanNormalization, true) { // Initialize the allocs request using the new optimized log entry format. // Determine the minimum number of updates, could be more if there // are multiple updates per node @@ -172,12 +172,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap for _, updateList := range result.NodeUpdate { for _, stoppedAlloc := range updateList { - req.AllocsStopped = append(req.AllocsStopped, &structs.Allocation{ - ID: stoppedAlloc.ID, - DesiredDescription: stoppedAlloc.DesiredDescription, - ClientStatus: stoppedAlloc.ClientStatus, - ModifyTime: now, - }) + req.AllocsStopped = append(req.AllocsStopped, normalizeStoppedAlloc(stoppedAlloc, now)) } } @@ -191,18 +186,14 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap for _, preemptions := range result.NodePreemptions { for _, preemptedAlloc := range preemptions { - req.NodePreemptions = append(req.NodePreemptions, &structs.Allocation{ - ID: preemptedAlloc.ID, - PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, - ModifyTime: now, - }) + req.NodePreemptions = append(req.NodePreemptions, normalizePreemptedAlloc(preemptedAlloc, now)) // Gather jobids to create follow up evals appendNamespacedJobID(preemptedJobIDs, preemptedAlloc) } } } else { - // Deprecated: This code path is deprecated and will only be used to support + // COMPAT 0.11: This branch is deprecated and will only be used to support // application of older log entries. Expected to be removed in a future version. // Determine the minimum number of updates, could be more if there @@ -270,6 +261,23 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap return future, nil } +func normalizePreemptedAlloc(preemptedAlloc *structs.Allocation, now int64) *structs.Allocation { + return &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, + ModifyTime: now, + } +} + +func normalizeStoppedAlloc(stoppedAlloc *structs.Allocation, now int64) *structs.Allocation { + return &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: stoppedAlloc.DesiredDescription, + ClientStatus: stoppedAlloc.ClientStatus, + ModifyTime: now, + } +} + func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *structs.Allocation) { id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} if _, ok := jobIDs[id]; !ok { @@ -318,11 +326,13 @@ func (p *planner) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture, func evaluatePlan(pool *EvaluatePool, snap *state.StateSnapshot, plan *structs.Plan, logger log.Logger) (*structs.PlanResult, error) { defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now()) - err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, plan.Job) + // Denormalize without the job + err := snap.DenormalizeAllocationsMap(plan.NodeUpdate, nil) if err != nil { return nil, err } - err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, plan.Job) + // Denormalize without the job + err = snap.DenormalizeAllocationsMap(plan.NodePreemptions, nil) if err != nil { return nil, err } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 619b7b9071d..538a7183aa4 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -63,7 +63,7 @@ func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { } } -// Deprecated: Tests the older unoptimized code path for applyPlan +// COMPAT 0.11: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { t.Parallel() s1 := TestServer(t, nil) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 2d7614dbdc1..f1a6fa0298a 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -673,6 +673,7 @@ type ApplyPlanResultsRequest struct { // to cause evictions or to assign new allocations. Both can be done // within a single transaction type AllocUpdateRequest struct { + // COMPAT 0.11 // Alloc is the list of new allocations to assign // Deprecated: Replaced with two separate slices, one containing stopped allocations // and another containing updated allocations diff --git a/nomad/util.go b/nomad/util.go index 8a6ff407f73..74dc41e04ec 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -14,10 +14,10 @@ import ( "github.com/hashicorp/serf/serf" ) -// MinVersionPlanDenormalization is the minimum version to support the -// denormalization of Plan in SubmitPlan, and the raft log entry committed +// MinVersionPlanNormalization is the minimum version to support the +// normalization of Plan in SubmitPlan, and the denormalization raft log entry committed // in ApplyPlanResultsRequest -var MinVersionPlanDenormalization = version.Must(version.NewVersion("0.9.1")) +var MinVersionPlanNormalization = version.Must(version.NewVersion("0.9.1")) // ensurePath is used to make sure a path exists func ensurePath(path string, dir bool) error { diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 3a4ec76ce58..4119d9be430 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -554,7 +554,7 @@ func TestServiceSched_EvenSpread(t *testing.T) { func TestServiceSched_JobRegister_Annotate(t *testing.T) { for _, allowPlanOptimization := range []bool{true, false} { t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) // Create some nodes for i := 0; i < 10; i++ { diff --git a/scheduler/testing.go b/scheduler/testing.go index dfe4f728639..6eb43794572 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -141,7 +141,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er req.AllocsStopped = allocsStopped req.AllocsUpdated = allocsUpdated } else { - // Deprecated: Handles unoptimized log format + // COMPAT 0.11: Handles unoptimized log format var allocs []*structs.Allocation allocs = append(allocs, allocsStopped...) allocs = append(allocs, allocsUpdated...) From 97686e371f61538d578e1a53038b348d8e81c994 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Fri, 8 Mar 2019 04:48:12 -0800 Subject: [PATCH 4/7] Remove allowPlanOptimization from schedulers --- nomad/fsm.go | 9 +- nomad/fsm_test.go | 2 +- nomad/job_endpoint.go | 3 +- nomad/plan_apply_test.go | 10 +- nomad/plan_normalization_test.go | 63 + nomad/server.go | 4 +- nomad/state/state_store_test.go | 10 +- nomad/structs/structs.go | 32 +- nomad/structs/structs_test.go | 55 +- nomad/worker.go | 8 +- nomad/worker_test.go | 18 +- scheduler/generic_sched.go | 32 +- scheduler/generic_sched_test.go | 7316 +++++++++++++++--------------- scheduler/scheduler.go | 6 +- scheduler/system_sched.go | 17 +- scheduler/system_sched_test.go | 3322 +++++++------- scheduler/testing.go | 36 +- scheduler/util_test.go | 12 +- 18 files changed, 5322 insertions(+), 5633 deletions(-) create mode 100644 nomad/plan_normalization_test.go diff --git a/nomad/fsm.go b/nomad/fsm.go index c54a40d86da..3c91b8f5c69 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -86,9 +86,6 @@ type nomadFSM struct { // new state store). Everything internal here is synchronized by the // Raft side, so doesn't need to lock this. stateLock sync.RWMutex - - // Reference to the server the FSM is running on - server *Server } // nomadSnapshot is used to provide a snapshot of the current @@ -124,7 +121,7 @@ type FSMConfig struct { } // NewFSMPath is used to construct a new FSM with a blank state -func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { +func NewFSM(config *FSMConfig) (*nomadFSM, error) { // Create a state store sconfig := &state.StateStoreConfig{ Logger: config.Logger, @@ -145,7 +142,6 @@ func NewFSM(config *FSMConfig, server *Server) (*nomadFSM, error) { timetable: NewTimeTable(timeTableGranularity, timeTableLimit), enterpriseAppliers: make(map[structs.MessageType]LogApplier, 8), enterpriseRestorers: make(map[SnapshotType]SnapshotRestorer, 8), - server: server, } // Register all the log applier functions @@ -1427,8 +1423,7 @@ func (n *nomadFSM) reconcileQueuedAllocations(index uint64) error { } snap.UpsertEvals(100, []*structs.Evaluation{eval}) // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(n.server.Members(), MinVersionPlanNormalization, true) - sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner, allowPlanOptimization) + sched, err := scheduler.NewScheduler(eval.Type, n.logger, snap, planner) if err != nil { return err } diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 06177a0831d..0c3a846a85f 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -56,7 +56,7 @@ func testFSM(t *testing.T) *nomadFSM { Logger: logger, Region: "global", } - fsm, err := NewFSM(fsmConfig, TestServer(t, nil)) + fsm, err := NewFSM(fsmConfig) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 8659929f9aa..1a5cdaf4300 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1216,8 +1216,7 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create the scheduler and run it - allowPlanOptimization := ServersMeetMinimumVersion(j.srv.Members(), MinVersionPlanNormalization, true) - sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner, allowPlanOptimization) + sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner) if err != nil { return err } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 538a7183aa4..f58f95bc1cf 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -265,13 +265,13 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { alloc := mock.Alloc() stoppedAlloc := mock.Alloc() stoppedAllocDiff := &structs.Allocation{ - ID: stoppedAlloc.ID, + ID: stoppedAlloc.ID, DesiredDescription: "Desired Description", - ClientStatus: structs.AllocClientStatusLost, + ClientStatus: structs.AllocClientStatusLost, } preemptedAlloc := mock.Alloc() preemptedAllocDiff := &structs.Allocation{ - ID: preemptedAlloc.ID, + ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) @@ -356,7 +356,7 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { assert.NotNil(updatedPreemptedAlloc) assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) assert.Equal(updatedPreemptedAlloc.DesiredDescription, - "Preempted by alloc ID " + preemptedAllocDiff.PreemptedByAllocation) + "Preempted by alloc ID "+preemptedAllocDiff.PreemptedByAllocation) assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) // Lookup the new deployment diff --git a/nomad/plan_normalization_test.go b/nomad/plan_normalization_test.go new file mode 100644 index 00000000000..ab5f5b5caa5 --- /dev/null +++ b/nomad/plan_normalization_test.go @@ -0,0 +1,63 @@ +package nomad + +import ( + "bytes" + "testing" + "time" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/assert" + "github.com/ugorji/go/codec" +) + +// Whenever this test is changed, care should be taken to ensure the older msgpack size +// is recalculated when new fields are introduced in ApplyPlanResultsRequest +func TestPlanNormalize(t *testing.T) { + // This size was calculated using the older ApplyPlanResultsRequest format, in which allocations + // didn't use OmitEmpty and only the job was normalized in the stopped and preempted allocs. + // The newer format uses OmitEmpty and uses a minimal set of fields for the diff of the + // stopped and preempted allocs. The file for the older format hasn't been checked in, because + // it's not a good idea to check-in a 20mb file to the git repo. + unoptimizedLogSize := 19460168 + + numUpdatedAllocs := 10000 + numStoppedAllocs := 8000 + numPreemptedAllocs := 2000 + mockAlloc := mock.Alloc() + mockAlloc.Job = nil + + mockUpdatedAllocSlice := make([]*structs.Allocation, numUpdatedAllocs) + for i := 0; i < numUpdatedAllocs; i++ { + mockUpdatedAllocSlice = append(mockUpdatedAllocSlice, mockAlloc) + } + + now := time.Now().UTC().UnixNano() + mockStoppedAllocSlice := make([]*structs.Allocation, numStoppedAllocs) + for i := 0; i < numStoppedAllocs; i++ { + mockStoppedAllocSlice = append(mockStoppedAllocSlice, normalizeStoppedAlloc(mockAlloc, now)) + } + + mockPreemptionAllocSlice := make([]*structs.Allocation, numPreemptedAllocs) + for i := 0; i < numPreemptedAllocs; i++ { + mockPreemptionAllocSlice = append(mockPreemptionAllocSlice, normalizePreemptedAlloc(mockAlloc, now)) + } + + // Create a plan result + applyPlanLogEntry := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + AllocsUpdated: mockUpdatedAllocSlice, + AllocsStopped: mockStoppedAllocSlice, + }, + NodePreemptions: mockPreemptionAllocSlice, + } + + handle := structs.MsgpackHandle + var buf bytes.Buffer + if err := codec.NewEncoder(&buf, handle).Encode(applyPlanLogEntry); err != nil { + t.Fatalf("Encoding failed: %v", err) + } + + optimizedLogSize := buf.Len() + assert.True(t, float64(optimizedLogSize)/float64(unoptimizedLogSize) < 0.62) +} diff --git a/nomad/server.go b/nomad/server.go index f8c4d2e4ca5..c35124c510c 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1078,7 +1078,7 @@ func (s *Server) setupRaft() error { Region: s.Region(), } var err error - s.fsm, err = NewFSM(fsmConfig, s) + s.fsm, err = NewFSM(fsmConfig) if err != nil { return err } @@ -1173,7 +1173,7 @@ func (s *Server) setupRaft() error { if err != nil { return fmt.Errorf("recovery failed to parse peers.json: %v", err) } - tmpFsm, err := NewFSM(fsmConfig, s) + tmpFsm, err := NewFSM(fsmConfig) if err != nil { return fmt.Errorf("recovery failed to make temp FSM: %v", err) } diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index af248a55729..0e0539b48d7 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -153,14 +153,14 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { stoppedAlloc := mock.Alloc() stoppedAlloc.Job = job stoppedAllocDiff := &structs.Allocation{ - ID: stoppedAlloc.ID, + ID: stoppedAlloc.ID, DesiredDescription: "desired desc", - ClientStatus: structs.AllocClientStatusLost, + ClientStatus: structs.AllocClientStatusLost, } preemptedAlloc := mock.Alloc() preemptedAlloc.Job = job preemptedAllocDiff := &structs.Allocation{ - ID: preemptedAlloc.ID, + ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } @@ -185,9 +185,9 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { AllocUpdateRequest: structs.AllocUpdateRequest{ AllocsUpdated: []*structs.Allocation{alloc}, AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, - Job: job, + Job: job, }, - EvalID: eval.ID, + EvalID: eval.ID, NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, } assert := assert.New(t) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f1a6fa0298a..d18d6b9c54f 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8247,7 +8247,7 @@ func (e *Evaluation) ShouldBlock() bool { // MakePlan is used to make a plan from the given evaluation // for a given Job -func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { +func (e *Evaluation) MakePlan(j *Job) *Plan { p := &Plan{ EvalID: e.ID, Priority: e.Priority, @@ -8255,7 +8255,6 @@ func (e *Evaluation) MakePlan(j *Job, allowPlanOptimization bool) *Plan { NodeUpdate: make(map[string][]*Allocation), NodeAllocation: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), - NormalizeAllocs: allowPlanOptimization, } if j != nil { p.AllAtOnce = j.AllAtOnce @@ -8377,9 +8376,6 @@ type Plan struct { // lower priority jobs that are preempted. Preempted allocations are marked // as evicted. NodePreemptions map[string][]*Allocation - - // Indicates whether allocs are normalized in the Plan - NormalizeAllocs bool `codec:"-"` } // AppendStoppedAlloc marks an allocation to be stopped. The clientStatus of the @@ -8473,23 +8469,21 @@ func (p *Plan) IsNoOp() bool { } func (p *Plan) NormalizeAllocations() { - if p.NormalizeAllocs { - for _, allocs := range p.NodeUpdate { - for i, alloc := range allocs { - allocs[i] = &Allocation{ - ID: alloc.ID, - DesiredDescription: alloc.DesiredDescription, - ClientStatus: alloc.ClientStatus, - } + for _, allocs := range p.NodeUpdate { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + DesiredDescription: alloc.DesiredDescription, + ClientStatus: alloc.ClientStatus, } } + } - for _, allocs := range p.NodePreemptions { - for i, alloc := range allocs { - allocs[i] = &Allocation{ - ID: alloc.ID, - PreemptedByAllocation: alloc.PreemptedByAllocation, - } + for _, allocs := range p.NodePreemptions { + for i, alloc := range allocs { + allocs[i] = &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: alloc.PreemptedByAllocation, } } } diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 2cd3922aebf..e574f51dc58 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -2842,13 +2842,12 @@ func TestTaskArtifact_Validate_Checksum(t *testing.T) { } } -func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { +func TestPlan_NormalizeAllocations(t *testing.T) { t.Parallel() plan := &Plan{ - NodeUpdate: make(map[string][]*Allocation), + NodeUpdate: make(map[string][]*Allocation), NodePreemptions: make(map[string][]*Allocation), } - plan.NormalizeAllocs = true stoppedAlloc := MockAlloc() desiredDesc := "Desired desc" plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) @@ -2873,45 +2872,6 @@ func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) } -func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsFalse(t *testing.T) { - t.Parallel() - plan := &Plan{ - NodeUpdate: make(map[string][]*Allocation), - NodePreemptions: make(map[string][]*Allocation), - } - plan.NormalizeAllocs = false - stoppedAlloc := MockAlloc() - desiredDesc := "Desired desc" - plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) - preemptedAlloc := MockAlloc() - preemptingAllocID := uuid.Generate() - plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) - - plan.NormalizeAllocations() - - actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] - expectedStoppedAlloc := new(Allocation) - *expectedStoppedAlloc = *stoppedAlloc - expectedStoppedAlloc.DesiredDescription = desiredDesc - expectedStoppedAlloc.DesiredStatus = AllocDesiredStatusStop - expectedStoppedAlloc.ClientStatus = AllocClientStatusLost - expectedStoppedAlloc.Job = nil - assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) - actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] - expectedPreemptedAlloc := &Allocation{ - ID: preemptedAlloc.ID, - PreemptedByAllocation: preemptingAllocID, - JobID: preemptedAlloc.JobID, - Namespace: preemptedAlloc.Namespace, - DesiredStatus: AllocDesiredStatusEvict, - DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), - AllocatedResources: preemptedAlloc.AllocatedResources, - TaskResources: preemptedAlloc.TaskResources, - SharedResources: preemptedAlloc.SharedResources, - } - assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) -} - func TestPlan_AppendStoppedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { t.Parallel() plan := &Plan{ @@ -2958,17 +2918,6 @@ func TestPlan_AppendPreemptedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { assert.Equal(t, expectedAlloc, appendedAlloc) } -func TestPlan_MsgPackTags(t *testing.T) { - t.Parallel() - planType := reflect.TypeOf(Plan{}) - - msgPackTags, _ := planType.FieldByName("_struct") - normalizeTag, _ := planType.FieldByName("NormalizeAllocs") - - assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) - assert.Equal(t, normalizeTag.Tag, reflect.StructTag(`codec:"-"`)) -} - func TestAllocation_MsgPackTags(t *testing.T) { t.Parallel() planType := reflect.TypeOf(Allocation{}) diff --git a/nomad/worker.go b/nomad/worker.go index 6cd8e0107fc..a50c58e68dd 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -284,8 +284,7 @@ func (w *Worker) invokeScheduler(eval *structs.Evaluation, token string) error { if eval.Type == structs.JobTypeCore { sched = NewCoreScheduler(w.srv, snap) } else { - allowPlanOptimization := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanDenormalization, true) - sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w, allowPlanOptimization) + sched, err = scheduler.NewScheduler(eval.Type, w.logger, snap, w) if err != nil { return fmt.Errorf("failed to instantiate scheduler: %v", err) } @@ -312,7 +311,10 @@ func (w *Worker) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, scheduler. plan.EvalToken = w.evalToken // Normalize stopped and preempted allocs before RPC - plan.NormalizeAllocations() + normalizePlan := ServersMeetMinimumVersion(w.srv.Members(), MinVersionPlanNormalization, true) + if normalizePlan { + plan.NormalizeAllocations() + } // Setup the request req := structs.PlanRequest{ diff --git a/nomad/worker_test.go b/nomad/worker_test.go index b2dc94f22b8..a03e739bfc3 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -19,11 +19,10 @@ import ( ) type NoopScheduler struct { - state scheduler.State - planner scheduler.Planner - eval *structs.Evaluation - allowPlanOptimization bool - err error + state scheduler.State + planner scheduler.Planner + eval *structs.Evaluation + err error } func (n *NoopScheduler) Process(eval *structs.Evaluation) error { @@ -38,11 +37,10 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { } func init() { - scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { + scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { n := &NoopScheduler{ - state: s, - planner: p, - allowPlanOptimization: allowPlanOptimization, + state: s, + planner: p, } return n } @@ -398,6 +396,7 @@ func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { s1 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.EnabledSchedulers = []string{structs.JobTypeService} + c.Build = "0.9.1" }) defer s1.Shutdown() testutil.WaitForLeader(t, s1.RPC) @@ -422,7 +421,6 @@ func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { EvalID: eval1.ID, NodeUpdate: make(map[string][]*structs.Allocation), NodePreemptions: make(map[string][]*structs.Allocation), - NormalizeAllocs: true, } desiredDescription := "desired desc" plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost) diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index b69b31a64be..376a826ee81 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -5,8 +5,8 @@ import ( "time" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" ) @@ -77,10 +77,6 @@ type GenericScheduler struct { planner Planner batch bool - // Temporary flag introduced till the code for sending/committing full allocs in the Plan can - // be safely removed - allowPlanOptimization bool - eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -98,25 +94,23 @@ type GenericScheduler struct { } // NewServiceScheduler is a factory function to instantiate a new service scheduler -func NewServiceScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewServiceScheduler(logger log.Logger, state State, planner Planner) Scheduler { s := &GenericScheduler{ - logger: logger.Named("service_sched"), - state: state, - planner: planner, - batch: false, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("service_sched"), + state: state, + planner: planner, + batch: false, } return s } // NewBatchScheduler is a factory function to instantiate a new batch scheduler -func NewBatchScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewBatchScheduler(logger log.Logger, state State, planner Planner) Scheduler { s := &GenericScheduler{ - logger: logger.Named("batch_sched"), - state: state, - planner: planner, - batch: true, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("batch_sched"), + state: state, + planner: planner, + batch: true, } return s } @@ -230,7 +224,7 @@ func (s *GenericScheduler) process() (bool, error) { s.followUpEvals = nil // Create a plan - s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) + s.plan = s.eval.MakePlan(s.job) if !s.batch { // Get any existing deployment diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 4119d9be430..70e54d23735 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -16,471 +16,614 @@ import ( "github.com/stretchr/testify/require" ) -func IsPlanOptimizedStr(allowPlanOptimization bool) string { - return fmt.Sprintf("Is plan optimized: %v", allowPlanOptimization) -} - -func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobRegister(t *testing.T) { + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct host and has count 1 higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Ensure the eval has no spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) + // Ensure different ports were used. + used := make(map[int]map[string]struct{}) + for _, alloc := range out { + for _, resource := range alloc.TaskResources { + for _, port := range resource.Networks[0].DynamicPorts { + nodeMap, ok := used[port.Value] + if !ok { + nodeMap = make(map[string]struct{}) + used[port.Value] = nodeMap + } + if _, ok := nodeMap[alloc.NodeID]; ok { + t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) } - used[alloc.NodeID] = struct{}{} + nodeMap[alloc.NodeID] = struct{}{} } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + } } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - rack := "rack2" - if i < 5 { - rack = "rack1" - } - node.Meta["rack"] = rack - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 8 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - RTarget: "2", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan allocated + plan := h.Plans[0] + planned := make(map[string]*structs.Allocation) + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + planned[alloc.ID] = alloc + } + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Update the job to force a rolling upgrade + updated := job.Copy() + updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 + noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State) + if err := h1.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure we have created only one new allocation + // Ensure a single plan + if len(h1.Plans) != 1 { + t.Fatalf("bad: %#v", h1.Plans) + } + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 10 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocations were placed on the same node as the older + // ones + for _, new := range newPlanned { + if new.PreviousAllocation == "" { + t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) + } - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + old, ok := planned[new.PreviousAllocation] + if !ok { + t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) + } + if new.NodeID != old.NodeID { + t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) + } + } +} - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 4 { - t.Fatalf("bad: %#v", plan) - } +func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { + h := NewHarness(t) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure all allocations placed - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Create a job with count 2 and disk as 60GB so that only one allocation + // can fit + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure each node was only used twice - used := make(map[string]uint64) - for _, alloc := range out { - if count, _ := used[alloc.NodeID]; count > 2 { - t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) - } - used[alloc.NodeID]++ - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } -} -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create some nodes - for i := 0; i < 2; i++ { - node := mock.Node() - node.Meta["ssd"] = "true" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Create a job that uses distinct property only on one task group. - job := mock.Job() - job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.ssd}", - }) + // Ensure the eval has a blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure the plan allocated only one allocation + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure only one allocation was placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } +func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { + h := NewHarness(t) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a job that uses distinct host and has count 1 higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure all allocations placed - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } -} -func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { - for _, allowPlanOptimization := range []bool{false, true} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - assert := assert.New(t) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create a job that uses distinct property over the node-id - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${node.unique.id}", - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 6; i++ { - node := mock.Node() - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Create some allocations - var allocs []*structs.Allocation - for i := 0; i < 3; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Update the count - job2 := job.Copy() - job2.TaskGroups[0].Count = 6 - assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") +func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { + h := NewHarness(t) - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + rack := "rack2" + if i < 5 { + rack = "rack1" + } + node.Meta["rack"] = rack + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - assert.Len(planned, 6, "Planned Allocations") + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 8 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - assert.Nil(err, "AllocsByJob") + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure all allocations placed - assert.Len(out, 6, "Placed Allocations") + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") } + + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 4 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Ensure each node was only used twice + used := make(map[string]uint64) + for _, alloc := range out { + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) + } + used[alloc.NodeID]++ + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// Test job registration with spread configured -func TestServiceSched_Spread(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - assert := assert.New(t) +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { + h := NewHarness(t) - start := uint8(100) - step := uint8(10) + // Create some nodes + for i := 0; i < 2; i++ { + node := mock.Node() + node.Meta["ssd"] = "true" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - for i := 0; i < 10; i++ { - name := fmt.Sprintf("%d%% in dc1", start) - t.Run(name, func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - remaining := uint8(100 - start) - // Create a job that uses spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - SpreadTarget: []*structs.SpreadTarget{ - { - Value: "dc1", - Percent: start, - }, - { - Value: "dc2", - Percent: remaining, - }, - }, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c - } - assert.Len(planned, 10, "Planned Allocations") - - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 10 - i - if i > 0 { - expectedCounts["dc2"] = i - } - require.Equal(t, expectedCounts, dcAllocsMap) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - start = start - step - } + // Create a job that uses distinct property only on one task group. + job := mock.Job() + job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.ssd}", }) + + job.TaskGroups[1].Name = "tg2" + job.TaskGroups[1].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 3 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } -// Test job registration with even spread across dc -func TestServiceSched_EvenSpread(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - assert := assert.New(t) +func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { + h := NewHarness(t) + assert := assert.New(t) + + // Create a job that uses distinct property over the node-id + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${node.unique.id}", + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 6; i++ { + node := mock.Node() + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + } + + // Create some allocations + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") + + // Update the count + job2 := job.Copy() + job2.TaskGroups[0].Count = 6 + assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + assert.Len(planned, 6, "Planned Allocations") + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + assert.Nil(err, "AllocsByJob") + + // Ensure all allocations placed + assert.Len(out, 6, "Placed Allocations") - h := NewHarness(t, allowPlanOptimization) - // Create a job that uses even spread over data center + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} + +// Test job registration with spread configured +func TestServiceSched_Spread(t *testing.T) { + assert := assert.New(t) + + start := uint8(100) + step := uint8(10) + + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%d%% in dc1", start) + t.Run(name, func(t *testing.T) { + h := NewHarness(t) + remaining := uint8(100 - start) + // Create a job that uses spread over data center job := mock.Job() job.Datacenters = []string{"dc1", "dc2"} job.TaskGroups[0].Count = 10 @@ -488,6 +631,16 @@ func TestServiceSched_EvenSpread(t *testing.T) { &structs.Spread{ Attribute: "${node.datacenter}", Weight: 100, + SpreadTarget: []*structs.SpreadTarget{ + { + Value: "dc1", + Percent: start, + }, + { + Value: "dc2", + Percent: remaining, + }, + }, }) assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") // Create some nodes, half in dc2 @@ -539,1094 +692,1117 @@ func TestServiceSched_EvenSpread(t *testing.T) { } assert.Len(planned, 10, "Planned Allocations") - // Expect even split allocs across datacenter expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 5 - expectedCounts["dc2"] = 5 - + expectedCounts["dc1"] = 10 - i + if i > 0 { + expectedCounts["dc2"] = i + } require.Equal(t, expectedCounts, dcAllocsMap) h.AssertEvalStatus(t, structs.EvalStatusComplete) }) + start = start - step } } -func TestServiceSched_JobRegister_Annotate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t) +// Test job registration with even spread across dc +func TestServiceSched_EvenSpread(t *testing.T) { + assert := assert.New(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + // Create a job that uses even spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Expect even split allocs across datacenter + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 5 + expectedCounts["dc2"] = 5 - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + require.Equal(t, expectedCounts, dcAllocsMap) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - h.AssertEvalStatus(t, structs.EvalStatusComplete) +func TestServiceSched_JobRegister_Annotate(t *testing.T) { + h := NewHarness(t) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expected := &structs.DesiredUpdates{Place: 10} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) - } - }) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } -} -func TestServiceSched_JobRegister_CountZero(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + expected := &structs.DesiredUpdates{Place: 10} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + } +} + +func TestServiceSched_JobRegister_CountZero(t *testing.T) { + h := NewHarness(t) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_AllocFail(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create NO nodes - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create NO nodes + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { - t.Fatalf("bad: %#v", metrics) - } + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { + t.Fatalf("bad: %#v", metrics) + } - // Check queued allocations - queued := outEval.QueuedAllocations["web"] - if queued != 10 { - t.Fatalf("expected queued: %v, actual: %v", 10, queued) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Check queued allocations + queued := outEval.QueuedAllocations["web"] + if queued != 10 { + t.Fatalf("expected queued: %v, actual: %v", 10, queued) } + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a full node - node := mock.Node() - node.ReservedResources = &structs.NodeReservedResources{ - Cpu: structs.NodeReservedCpuResources{ - CpuShares: node.NodeResources.Cpu.CpuShares, - }, - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h := NewHarness(t) - // Create an ineligible node - node2 := mock.Node() - node2.Attributes["kernel.name"] = "windows" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + // Create a full node + node := mock.Node() + node.ReservedResources = &structs.NodeReservedResources{ + Cpu: structs.NodeReservedCpuResources{ + CpuShares: node.NodeResources.Cpu.CpuShares, + }, + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a jobs - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create an ineligible node + node2 := mock.Node() + node2.Attributes["kernel.name"] = "windows" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a jobs + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan has created a follow up eval. - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - created := h.CreateEvals[0] - if created.Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", created) - } + // Ensure the plan has created a follow up eval. + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - classes := created.ClassEligibility - if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { - t.Fatalf("bad: %#v", classes) - } + created := h.CreateEvals[0] + if created.Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", created) + } - if created.EscapedComputedClass { - t.Fatalf("bad: %#v", created) - } + classes := created.ClassEligibility + if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { + t.Fatalf("bad: %#v", classes) + } - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } + if created.EscapedComputedClass { + t.Fatalf("bad: %#v", created) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { - t.Fatalf("bad: %#v", metrics) - } + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { + t.Fatalf("bad: %#v", metrics) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create one node - node := mock.Node() - node.NodeClass = "class_0" - noErr(t, node.ComputeClass()) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h := NewHarness(t) - // Create a job that constrains on a node class - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].Constraints = append(job.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "class_0", - Operand: "=", - }, - ) - tg2 := job.TaskGroups[0].Copy() - tg2.Name = "web2" - tg2.Constraints[1].RTarget = "class_1" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create one node + node := mock.Node() + node.NodeClass = "class_0" + noErr(t, node.ComputeClass()) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job that constrains on a node class + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].Constraints = append(job.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "class_0", + Operand: "=", + }, + ) + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "web2" + tg2.Constraints[1].RTarget = "class_1" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 2 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure two allocations placed - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 2 { + t.Fatalf("bad: %#v", plan) + } - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] + // Ensure two allocations placed + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] - // Ensure the plan failed to alloc one tg - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } - metrics, ok := outEval.FailedTGAllocs[tg2.Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } + // Ensure the plan failed to alloc one tg + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } - // Check the coalesced failures - if metrics.CoalescedFailures != tg2.Count-1 { - t.Fatalf("bad: %#v", metrics) - } + metrics, ok := outEval.FailedTGAllocs[tg2.Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Check the coalesced failures + if metrics.CoalescedFailures != tg2.Count-1 { + t.Fatalf("bad: %#v", metrics) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test just ensures the scheduler handles the eval type to avoid // regressions. func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + h := NewHarness(t) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerMaxPlans, - JobID: job.ID, - } + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerMaxPlans, + JobID: job.ID, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_Plan_Partial_Progress(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h := NewHarness(t) - // Create a job with a high resource ask so that all the allocations can't - // be placed on a single node. - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a job with a high resource ask so that all the allocations can't + // be placed on a single node. + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure only one allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } + // Ensure only one allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_EvaluateBlockedEval(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure that the eval was reblocked - if len(h.ReblockEvals) != 1 { - t.Fatalf("bad: %#v", h.ReblockEvals) - } - if h.ReblockEvals[0].ID != eval.ID { - t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) - } + // Ensure that the eval was reblocked + if len(h.ReblockEvals) != 1 { + t.Fatalf("bad: %#v", h.ReblockEvals) + } + if h.ReblockEvals[0].ID != eval.ID { + t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) + } - // Ensure the eval status was not updated - if len(h.Evals) != 0 { - t.Fatalf("Existing eval should not have status set") - } - }) + // Ensure the eval status was not updated + if len(h.Evals) != 0 { + t.Fatalf("Existing eval should not have status set") } } func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job and set the task group count to zero. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the eval has no spawned blocked eval - if len(h.Evals) != 1 { - t.Fatalf("bad: %#v", h.Evals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure the eval has no spawned blocked eval + if len(h.Evals) != 1 { + t.Fatalf("bad: %#v", h.Evals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure the eval was not reblocked - if len(h.ReblockEvals) != 0 { - t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") - } + // Ensure the eval was not reblocked + if len(h.ReblockEvals) != 0 { + t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure queued allocations is zero - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued: %v, actual: %v", 0, queued) - } - }) + // Ensure queued allocations is zero + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued: %v, actual: %v", 0, queued) } } func TestServiceSched_JobModify(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job + job2 := mock.Job() + job2.ID = job.ID - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // Have a single node and submit a job. Increment the count such that all fit // on the node but the node doesn't have enough resources to fit the new count + // 1. This tests that we properly discount the resources of existing allocs. func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create one node - node := mock.Node() - node.NodeResources.Cpu.CpuShares = 1000 - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create one node + node := mock.Node() + node.NodeResources.Cpu.CpuShares = 1000 + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with one allocation - job := mock.Job() - job.TaskGroups[0].Tasks[0].Resources.CPU = 256 - job2 := job.Copy() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with one allocation + job := mock.Job() + job.TaskGroups[0].Tasks[0].Resources.CPU = 256 + job2 := job.Copy() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job to count 3 - job2.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + var allocs []*structs.Allocation + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the job to count 3 + job2.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan didn't evicted the alloc - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan had no failures - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - if outEval == nil || len(outEval.FailedTGAllocs) != 0 { - t.Fatalf("bad: %#v", outEval) - } + // Ensure the plan didn't evicted the alloc + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } + // Ensure the plan had no failures + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + if outEval == nil || len(outEval.FailedTGAllocs) != 0 { + t.Fatalf("bad: %#v", outEval) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 3 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobModify_CountZero(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the job to be count zero - job2 := mock.Job() - job2.ID = job.ID - job2.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Update the job to be count zero + job2 := mock.Job() + job2.ID = job.ID + job2.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan didn't allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan didn't allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobModify_Rolling(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != desiredUpdates { - t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != desiredUpdates { + t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } - }) + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) } } @@ -1635,2578 +1811,2206 @@ func TestServiceSched_JobModify_Rolling(t *testing.T) { // allocations as this allows us to assert that destructive changes are done // first. func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a node and clear the reserved resources - node := mock.Node() - node.ReservedResources = nil - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a resource ask that is the same as the resources available on the - // node - cpu := node.NodeResources.Cpu.CpuShares - mem := node.NodeResources.Memory.MemoryMB - - request := &structs.Resources{ - CPU: int(cpu), - MemoryMB: int(mem), - } - allocated := &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: cpu, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: mem, - }, - }, - }, - } + h := NewHarness(t) - // Generate a fake job with one alloc that consumes the whole node - job := mock.Job() - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Tasks[0].Resources = request - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.AllocatedResources = allocated - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Update the job to place more versions of the task group, drop the count - // and force destructive updates - job2 := job.Copy() - job2.TaskGroups[0].Count = 5 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources + // Create a node and clear the reserved resources + node := mock.Node() + node.ReservedResources = nil + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Create a resource ask that is the same as the resources available on the + // node + cpu := node.NodeResources.Cpu.CpuShares + mem := node.NodeResources.Memory.MemoryMB - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + request := &structs.Resources{ + CPU: int(cpu), + MemoryMB: int(mem), + } + allocated := &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: cpu, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: mem, + }, + }, + }, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with one alloc that consumes the whole node + job := mock.Job() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Resources = request + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + alloc := mock.Alloc() + alloc.AllocatedResources = allocated + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Update the job to place more versions of the task group, drop the count + // and force destructive updates + job2 := job.Copy() + job2.TaskGroups[0].Count = 5 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 1 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 5 { - t.Fatalf("bad: %#v", plan) - } + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } - }) + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 1 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 5 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) } } func TestServiceSched_JobModify_Canaries(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t) - h := NewHarness(t, allowPlanOptimization) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 2 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - Canary: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 2 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + Canary: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted nothing - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) - } + // Ensure the plan evicted nothing + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - for _, canary := range planned { - if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { - t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) - } - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { - t.Fatalf("bad: %#v", state) - } + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { + t.Fatalf("bad: %#v", state) + } - // Assert the canaries were added to the placed list - if len(state.PlacedCanaries) != desiredUpdates { - t.Fatalf("bad: %#v", state) - } - }) + // Assert the canaries were added to the placed list + if len(state.PlacedCanaries) != desiredUpdates { + t.Fatalf("bad: %#v", state) } } func TestServiceSched_JobModify_InPlace(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t) - h := NewHarness(t, allowPlanOptimization) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Generate a fake job with allocations and create an older deployment + job := mock.Job() + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Generate a fake job with allocations and create an older deployment - job := mock.Job() - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Create allocs that are part of the old deployment + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create allocs that are part of the old deployment - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = d.ID - alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) - } - } + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) } + } + } - // Verify the deployment id was changed and health cleared - for _, alloc := range out { - if alloc.DeploymentID == d.ID { - t.Fatalf("bad: deployment id not cleared") - } else if alloc.DeploymentStatus != nil { - t.Fatalf("bad: deployment status not cleared") - } - } - }) + // Verify the deployment id was changed and health cleared + for _, alloc := range out { + if alloc.DeploymentID == d.ID { + t.Fatalf("bad: deployment id not cleared") + } else if alloc.DeploymentStatus != nil { + t.Fatalf("bad: deployment status not cleared") + } } } func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - node.Meta["rack"] = fmt.Sprintf("rack%d", i) - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + node.Meta["rack"] = fmt.Sprintf("rack%d", i) + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - oldJob := job.Copy() - oldJob.JobModifyIndex -= 1 - oldJob.TaskGroups[0].Count = 4 + oldJob := job.Copy() + oldJob.JobModifyIndex -= 1 + oldJob.TaskGroups[0].Count = 4 - // Place 4 of 10 - var allocs []*structs.Allocation - for i := 0; i < 4; i++ { - alloc := mock.Alloc() - alloc.Job = oldJob - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Place 4 of 10 + var allocs []*structs.Allocation + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = oldJob + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", planned) - } + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", planned) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobDeregister_Purged(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Generate a fake job with allocations - job := mock.Job() + h := NewHarness(t) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations + job := mock.Job() - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all nodes - if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan evicted all nodes + if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - if alloc.Job == nil { - t.Fatalf("bad: %#v", alloc) - } - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + if alloc.Job == nil { + t.Fatalf("bad: %#v", alloc) + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobDeregister_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) + h := NewHarness(t) + require := require.New(t) - // Generate a fake job with allocations - job := mock.Job() - job.Stop = true - require.NoError(h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Stop = true + require.NoError(h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a summary where the queued allocs are set as we want to assert - // they get zeroed out. - summary := mock.JobSummary(job.ID) - web := summary.Summary["web"] - web.Queued = 2 - require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) + // Create a summary where the queued allocs are set as we want to assert + // they get zeroed out. + summary := mock.JobSummary(job.ID) + web := summary.Summary["web"] + web.Queued = 2 + require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - require.NoError(h.Process(NewServiceScheduler, eval)) + // Process the evaluation + require.NoError(h.Process(NewServiceScheduler, eval)) - // Ensure a single plan - require.Len(h.Plans, 1) - plan := h.Plans[0] + // Ensure a single plan + require.Len(h.Plans, 1) + plan := h.Plans[0] - // Ensure the plan evicted all nodes - require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) + // Ensure the plan evicted all nodes + require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(err) - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - require.NotNil(alloc.Job) - } + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + require.NotNil(alloc.Job) + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - require.Empty(out) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + require.Empty(out) - // Assert the job summary is cleared out - sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) - require.NoError(err) - require.NotNil(sout) - require.Contains(sout.Summary, "web") - webOut := sout.Summary["web"] - require.Zero(webOut.Queued) + // Assert the job summary is cleared out + sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) + require.NoError(err) + require.NotNil(sout) + require.Contains(sout.Summary, "web") + webOut := sout.Summary["web"] + require.Zero(webOut.Queued) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDown(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h := NewHarness(t) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Register a node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Cover each terminal case and ensure it doesn't change to lost - allocs[7].DesiredStatus = structs.AllocDesiredStatusRun - allocs[7].ClientStatus = structs.AllocClientStatusLost - allocs[8].DesiredStatus = structs.AllocDesiredStatusRun - allocs[8].ClientStatus = structs.AllocClientStatusFailed - allocs[9].DesiredStatus = structs.AllocDesiredStatusRun - allocs[9].ClientStatus = structs.AllocClientStatusComplete - - // Mark some allocs as running - for i := 0; i < 4; i++ { - out := allocs[i] - out.ClientStatus = structs.AllocClientStatusRunning - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } - // Mark appropriate allocs for migration - for i := 0; i < 7; i++ { - out := allocs[i] - out.DesiredTransition.Migrate = helper.BoolToPtr(true) - } + // Cover each terminal case and ensure it doesn't change to lost + allocs[7].DesiredStatus = structs.AllocDesiredStatusRun + allocs[7].ClientStatus = structs.AllocClientStatusLost + allocs[8].DesiredStatus = structs.AllocDesiredStatusRun + allocs[8].ClientStatus = structs.AllocClientStatusFailed + allocs[9].DesiredStatus = structs.AllocDesiredStatusRun + allocs[9].ClientStatus = structs.AllocClientStatusComplete + + // Mark some allocs as running + for i := 0; i < 4; i++ { + out := allocs[i] + out.ClientStatus = structs.AllocClientStatusRunning + } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Mark appropriate allocs for migration + for i := 0; i < 7; i++ { + out := allocs[i] + out.DesiredTransition.Migrate = helper.BoolToPtr(true) + } - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Test the scheduler marked all non-terminal allocations as lost - if len(plan.NodeUpdate[node.ID]) != 7 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - for _, out := range plan.NodeUpdate[node.ID] { - if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad alloc: %#v", out) - } - } + // Test the scheduler marked all non-terminal allocations as lost + if len(plan.NodeUpdate[node.ID]) != 7 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + for _, out := range plan.NodeUpdate[node.ID] { + if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad alloc: %#v", out) + } } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeUpdate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Mark some allocs as running - ws := memdb.NewWatchSet() - for i := 0; i < 4; i++ { - out, _ := h.State.AllocByID(ws, allocs[i].ID) - out.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) - } + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation which won't trigger any new placements - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Mark some allocs as running + ws := memdb.NewWatchSet() + for i := 0; i < 4; i++ { + out, _ := h.State.AllocByID(ws, allocs[i].ID) + out.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) - } + // Create a mock evaluation which won't trigger any new placements + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDrain(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDrain_Down(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Set the desired state of the allocs to stop - var stop []*structs.Allocation - for i := 0; i < 6; i++ { - newAlloc := allocs[i].Copy() - newAlloc.ClientStatus = structs.AllocDesiredStatusStop - newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - stop = append(stop, newAlloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) - - // Mark some of the allocations as running - var running []*structs.Allocation - for i := 4; i < 6; i++ { - newAlloc := stop[i].Copy() - newAlloc.ClientStatus = structs.AllocClientStatusRunning - running = append(running, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) - - // Mark some of the allocations as complete - var complete []*structs.Allocation - for i := 6; i < 10; i++ { - newAlloc := allocs[i].Copy() - newAlloc.TaskStates = make(map[string]*structs.TaskState) - newAlloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - newAlloc.ClientStatus = structs.AllocClientStatusComplete - complete = append(complete, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + // Set the desired state of the allocs to stop + var stop []*structs.Allocation + for i := 0; i < 6; i++ { + newAlloc := allocs[i].Copy() + newAlloc.ClientStatus = structs.AllocDesiredStatusStop + newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + stop = append(stop, newAlloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) + + // Mark some of the allocations as running + var running []*structs.Allocation + for i := 4; i < 6; i++ { + newAlloc := stop[i].Copy() + newAlloc.ClientStatus = structs.AllocClientStatusRunning + running = append(running, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) + + // Mark some of the allocations as complete + var complete []*structs.Allocation + for i := 6; i < 10; i++ { + newAlloc := allocs[i].Copy() + newAlloc.TaskStates = make(map[string]*structs.TaskState) + newAlloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + newAlloc.ClientStatus = structs.AllocClientStatusComplete + complete = append(complete, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 6 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure that all the allocations which were in running or pending state - // has been marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - sort.Strings(lostAllocs) + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 6 { + t.Fatalf("bad: %#v", plan) + } - var expectedLostAllocs []string - for i := 0; i < 6; i++ { - expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) - } - sort.Strings(expectedLostAllocs) + // Ensure that all the allocations which were in running or pending state + // has been marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + sort.Strings(lostAllocs) - if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { - t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) - } + var expectedLostAllocs []string + for i := 0; i < 6; i++ { + expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) + } + sort.Strings(expectedLostAllocs) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { + t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + h := NewHarness(t) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Register a draining node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } - }) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) } -} - -func TestServiceSched_RetryLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - h.Planner = &RejectPlan{h} - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) - }) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, } -} - -func TestServiceSched_Reschedule_OnceNow(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: 5 * time.Second, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) - - // Mark this alloc as failed again, should not get rescheduled - newAlloc.ClientStatus = structs.AllocClientStatusFailed - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err = h.Process(NewServiceScheduler, eval) - assert.Nil(err) - // Verify no new allocs were created this time - out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(3, len(out)) - }) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } -} - -// Tests that alloc reschedulable at a future time creates a follow up eval -func TestServiceSched_Reschedule_Later(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - delayDuration := 15 * time.Second - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: delayDuration, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now}} - failedAllocID := allocs[1].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify no new allocs were created - require.Equal(2, len(out)) - - // Verify follow up eval was created for the failed alloc - alloc, err := h.State.AllocByID(ws, failedAllocID) - require.Nil(err) - require.NotEmpty(alloc.FollowupEvalID) - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { - t.Fatalf("bad: %#v", h.CreateEvals) - } - followupEval := h.CreateEvals[0] - require.Equal(now.Add(delayDuration), followupEval.WaitUntil) - }) + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) } } -func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - maxRestartAttempts := 3 - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: maxRestartAttempts, - Interval: 30 * time.Minute, - Delay: 5 * time.Second, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.ClientStatus = structs.AllocClientStatusRunning - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - expectedNumAllocs := 3 - expectedNumReschedTrackers := 1 - - failedAllocId := allocs[1].ID - failedNodeID := allocs[1].NodeID - - assert := assert.New(t) - for i := 0; i < maxRestartAttempts; i++ { - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - noErr(t, err) - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that a new allocation got created with its restart tracker info - assert.Equal(expectedNumAllocs, len(out)) - - // Find the new alloc with ClientStatusPending - var pendingAllocs []*structs.Allocation - var prevFailedAlloc *structs.Allocation - - for _, alloc := range out { - if alloc.ClientStatus == structs.AllocClientStatusPending { - pendingAllocs = append(pendingAllocs, alloc) - } - if alloc.ID == failedAllocId { - prevFailedAlloc = alloc - } - } - assert.Equal(1, len(pendingAllocs)) - newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) - - // Verify the previous NodeID in the most recent reschedule event - reschedEvents := newAlloc.RescheduleTracker.Events - assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) - assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) - - // Verify that the next alloc of the failed alloc is the newly rescheduled alloc - assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) - - // Mark this alloc as failed again - newAlloc.ClientStatus = structs.AllocClientStatusFailed - newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-12 * time.Second), - FinishedAt: now.Add(-10 * time.Second)}} - - failedAllocId = newAlloc.ID - failedNodeID = newAlloc.NodeID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expectedNumAllocs += 1 - expectedNumReschedTrackers += 1 - } - - // Process last eval again, should not reschedule - err := h.Process(NewServiceScheduler, eval) - assert.Nil(err) +func TestServiceSched_RetryLimit(t *testing.T) { + h := NewHarness(t) + h.Planner = &RejectPlan{h} - // Verify no new allocs were created because restart attempts were exhausted - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts - }) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } -} - -// Tests that old reschedule attempts are pruned -func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - DelayFunction: "exponential", - MaxDelay: 1 * time.Hour, - Delay: 5 * time.Second, - Unlimited: true, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - now := time.Now() - // Mark allocations as failed with restart info - allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-15 * time.Minute)}} - allocs[1].ClientStatus = structs.AllocClientStatusFailed + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - allocs[1].RescheduleTracker = &structs.RescheduleTracker{ - Events: []*structs.RescheduleEvent{ - {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), - PrevAllocID: uuid.Generate(), - PrevNodeID: uuid.Generate(), - Delay: 5 * time.Second, - }, - {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 10 * time.Second, - }, - {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 20 * time.Second, - }, - {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 40 * time.Second, - }, - {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 80 * time.Second, - }, - {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 160 * time.Second, - }, - }, - } - expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] - expectedDelay := 320 * time.Second - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) +} - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - // Verify that the new alloc copied the last 5 reschedule attempts - assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) +func TestServiceSched_Reschedule_OnceNow(t *testing.T) { + h := NewHarness(t) - mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] - // Verify that the failed alloc ID is in the most recent reschedule event - assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) - // Verify that the delay value was captured correctly - assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) - }) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } -} -// Tests that deployments with failed allocs result in placements as long as the -// deployment is running. -func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - for _, failedDeployment := range []bool{false, true} { - t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and a reschedule policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) - - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - if failedDeployment { - deployment.Status = structs.DeploymentStatusFailed - } - - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed in the past - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", - StartedAt: time.Now().Add(-12 * time.Hour), - FinishedAt: time.Now().Add(-10 * time.Hour)}} - allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) - - if failedDeployment { - // Verify no plan created - require.Len(h.Plans, 0) - } else { - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - } - }) - } - }) + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: 5 * time.Second, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", } -} + tgName := job.TaskGroups[0].Name + now := time.Now() -func TestBatchSched_Run_CompleteAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan as it should be a no-op - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) - // Ensure no allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Mark this alloc as failed again, should not get rescheduled + newAlloc.ClientStatus = structs.AllocClientStatusFailed - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err = h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created this time + out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(3, len(out)) + } -func TestBatchSched_Run_FailedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +// Tests that alloc reschedulable at a future time creates a follow up eval +func TestServiceSched_Reschedule_Later(t *testing.T) { + h := NewHarness(t) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + delayDuration := 15 * time.Second + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: delayDuration, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - tgName := job.TaskGroups[0].Name - now := time.Now() - - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now}} + failedAllocID := allocs[1].ID - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure a replacement alloc was placed. - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } + // Verify no new allocs were created + require.Equal(2, len(out)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Verify follow up eval was created for the failed alloc + alloc, err := h.State.AllocByID(ws, failedAllocID) + require.Nil(err) + require.NotEmpty(alloc.FollowupEvalID) + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { + t.Fatalf("bad: %#v", h.CreateEvals) } + followupEval := h.CreateEvals[0] + require.Equal(now.Add(delayDuration), followupEval.WaitUntil) } -func TestBatchSched_Run_LostAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { + h := NewHarness(t) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - job.ID = "my-job" - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + maxRestartAttempts := 3 + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: maxRestartAttempts, + Interval: 30 * time.Minute, + Delay: 5 * time.Second, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() - // Desired = 3 - // Mark one as lost and then schedule - // [(0, run, running), (1, run, running), (1, stop, lost)] + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create two running allocations - var allocs []*structs.Allocation - for i := 0; i <= 1; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) - } + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[1]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + expectedNumAllocs := 3 + expectedNumReschedTrackers := 1 - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + failedAllocId := allocs[1].ID + failedNodeID := allocs[1].NodeID - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + assert := assert.New(t) + for i := 0; i < maxRestartAttempts; i++ { + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + noErr(t, err) - // Ensure a replacement alloc was placed. - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Assert that we have the correct number of each alloc name - expected := map[string]int{ - "my-job.web[0]": 1, - "my-job.web[1]": 2, - "my-job.web[2]": 1, - } - actual := make(map[string]int, 3) - for _, alloc := range out { - actual[alloc.Name] += 1 - } - require.Equal(t, actual, expected) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} + // Verify that a new allocation got created with its restart tracker info + assert.Equal(expectedNumAllocs, len(out)) -func TestServiceSched_JobRegister(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + // Find the new alloc with ClientStatusPending + var pendingAllocs []*structs.Allocation + var prevFailedAlloc *structs.Allocation - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + for _, alloc := range out { + if alloc.ClientStatus == structs.AllocClientStatusPending { + pendingAllocs = append(pendingAllocs, alloc) } + if alloc.ID == failedAllocId { + prevFailedAlloc = alloc + } + } + assert.Equal(1, len(pendingAllocs)) + newAlloc := pendingAllocs[0] + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) + + // Verify the previous NodeID in the most recent reschedule event + reschedEvents := newAlloc.RescheduleTracker.Events + assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) + assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) + + // Verify that the next alloc of the failed alloc is the newly rescheduled alloc + assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) + + // Mark this alloc as failed again + newAlloc.ClientStatus = structs.AllocClientStatusFailed + newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-12 * time.Second), + FinishedAt: now.Add(-10 * time.Second)}} + + failedAllocId = newAlloc.ID + failedNodeID = newAlloc.NodeID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + expectedNumAllocs += 1 + expectedNumReschedTrackers += 1 + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Process last eval again, should not reschedule + err := h.Process(NewServiceScheduler, eval) + assert.Nil(err) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Verify no new allocs were created because restart attempts were exhausted + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts +} - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +// Tests that old reschedule attempts are pruned +func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { + h := NewHarness(t) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + DelayFunction: "exponential", + MaxDelay: 1 * time.Hour, + Delay: 5 * time.Second, + Unlimited: true, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + now := time.Now() + // Mark allocations as failed with restart info + allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-15 * time.Minute)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed + + allocs[1].RescheduleTracker = &structs.RescheduleTracker{ + Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + Delay: 5 * time.Second, + }, + {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 10 * time.Second, + }, + {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 20 * time.Second, + }, + {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 40 * time.Second, + }, + {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 80 * time.Second, + }, + {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 160 * time.Second, + }, + }, + } + expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] + expectedDelay := 320 * time.Second + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - // Ensure the eval has no spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } - // Ensure different ports were used. - used := make(map[int]map[string]struct{}) - for _, alloc := range out { - for _, resource := range alloc.TaskResources { - for _, port := range resource.Networks[0].DynamicPorts { - nodeMap, ok := used[port.Value] - if !ok { - nodeMap = make(map[string]struct{}) - used[port.Value] = nodeMap - } - if _, ok := nodeMap[alloc.NodeID]; ok { - t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) - } - nodeMap[alloc.NodeID] = struct{}{} - } - } - } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + // Verify that the new alloc copied the last 5 reschedule attempts + assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) - } -} + mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] + // Verify that the failed alloc ID is in the most recent reschedule event + assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) + // Verify that the delay value was captured correctly + assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) -func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +} +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t) + require := require.New(t) // Create some nodes + var nodes []*structs.Node for i := 0; i < 10; i++ { node := mock.Node() + nodes = append(nodes, node) noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Create a job + // Generate a fake job with allocations and a reschedule policy. job := mock.Job() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed } - // Ensure the plan allocated - plan := h.Plans[0] - planned := make(map[string]*structs.Allocation) - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - planned[alloc.ID] = alloc - } - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = deployment.ID + allocs = append(allocs, alloc) } + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) - // Update the job to force a rolling upgrade - updated := job.Copy() - updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 - noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ + // Create a mock evaluation + eval := &structs.Evaluation{ Namespace: structs.DefaultNamespace, ID: uuid.Generate(), - Priority: job.Priority, + Priority: 50, TriggeredBy: structs.EvalTriggerNodeUpdate, JobID: job.ID, Status: structs.EvalStatusPending, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - if err := h1.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure we have created only one new allocation - // Ensure a single plan - if len(h1.Plans) != 1 { - t.Fatalf("bad: %#v", h1.Plans) - } - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 10 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocations were placed on the same node as the older - // ones - for _, new := range newPlanned { - if new.PreviousAllocation == "" { - t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) - } + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - old, ok := planned[new.PreviousAllocation] - if !ok { - t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) + // Process the evaluation + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) } - if new.NodeID != old.NodeID { - t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) } } }) } } -func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) +func TestBatchSched_Run_CompleteAlloc(t *testing.T) { + h := NewHarness(t) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job with count 2 and disk as 60GB so that only one allocation - // can fit - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - ID: uuid.Generate(), - } + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure no plan as it should be a no-op + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure the eval has a blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } + // Ensure no allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} - // Ensure the plan allocated only one allocation - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } +func TestBatchSched_Run_FailedAlloc(t *testing.T) { + h := NewHarness(t) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Ensure only one allocation was placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } + + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) +} + +func TestBatchSched_Run_LostAlloc(t *testing.T) { + h := NewHarness(t) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.Job() + job.ID = "my-job" + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Desired = 3 + // Mark one as lost and then schedule + // [(0, run, running), (1, run, running), (1, stop, lost)] + + // Create two running allocations + var allocs []*structs.Allocation + for i := 0; i <= 1; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[1]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Assert that we have the correct number of each alloc name + expected := map[string]int{ + "my-job.web[0]": 1, + "my-job.web[1]": 2, + "my-job.web[2]": 1, } + actual := make(map[string]int, 3) + for _, alloc := range out { + actual[alloc.Name] += 1 + } + require.Equal(t, actual, expected) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - tgName := job.TaskGroups[0].Name - now := time.Now() - - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + tgName := job.TaskGroups[0].Name + now := time.Now() - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 1 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } - }) + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 1 { + t.Fatalf("expected: %v, actual: %v", 1, queued) } } func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a successful alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + h := NewHarness(t) - // Create a mock evaluation to rerun the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a successful alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Create a mock evaluation to rerun the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure no replacement alloc was placed. - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no replacement alloc was placed. + if len(out) != 1 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test checks that terminal allocations that receive an in-place updated // are not added to the plan func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to trigger the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to trigger the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans[0]) - } - }) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans[0]) } } // This test ensures that terminal jobs from older versions are ignored. func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the job - job2 := mock.Job() - job2.ID = job.ID - job2.Type = structs.JobTypeBatch - job2.Version++ - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the job + job2 := mock.Job() + job2.ID = job.ID + job2.Type = structs.JobTypeBatch + job2.Version++ + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - allocs = nil - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job2 - alloc.JobID = job2.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, + allocs = nil + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job2 + alloc.JobID = job2.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, }, - } - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + }, + }, + } + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - }) + // Ensure a plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) } } // This test asserts that an allocation from an old job that is running on a // drained node is cleaned up. func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a running alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create an update job - job2 := job.Copy() - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - job2.Version++ - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + h := NewHarness(t) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a running alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create an update job + job2 := job.Copy() + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + job2.Version++ + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - plan := h.Plans[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan evicted 1 - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan places 1 - if len(plan.NodeAllocation[node2.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + plan := h.Plans[0] + + // Ensure the plan evicted 1 + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) } + + // Ensure the plan places 1 + if len(plan.NodeAllocation[node2.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test asserts that an allocation from a job that is complete on a // drained node is ignored up. func TestBatchSched_NodeDrain_Complete(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = make(map[string]*structs.TaskState) - alloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + h := NewHarness(t) + + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = make(map[string]*structs.TaskState) + alloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This is a slightly odd test but it ensures that we handle a scale down of a @@ -4295,318 +4099,303 @@ func TestBatchSched_ScaleDown_SameName(t *testing.T) { } func TestGenericSched_ChainedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - job1 := mock.Job() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - job1.TaskGroups[0].Count = 12 - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State) + job1 := mock.Job() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + job1.TaskGroups[0].Count = 12 + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewServiceScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewServiceScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue - } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + plan := h1.Plans[0] - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) - } - }) + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } + + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) } } func TestServiceSched_NodeDrain_Sticky(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create an alloc on the draining node - alloc := mock.Alloc() - alloc.Name = "my-job.web[0]" - alloc.NodeID = node.ID - alloc.Job.TaskGroups[0].Count = 1 - alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: alloc.Job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } + h := NewHarness(t) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create an alloc on the draining node + alloc := mock.Alloc() + alloc.Name = "my-job.web[0]" + alloc.NodeID = node.ID + alloc.Job.TaskGroups[0].Count = 1 + alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: alloc.Job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan didn't create any new allocations - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan didn't create any new allocations + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is stopped, the scheduler properly cancels // an outstanding deployment. func TestServiceSched_CancelDeployment_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) + h := NewHarness(t) - // Generate a fake job - job := mock.Job() - job.JobModifyIndex = job.CreateIndex + 1 - job.ModifyIndex = job.CreateIndex + 1 - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a deployment - d := mock.Deployment() - d.JobID = job.ID - d.JobCreateIndex = job.CreateIndex - d.JobModifyIndex = job.JobModifyIndex - 1 - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Generate a fake job + job := mock.Job() + job.JobModifyIndex = job.CreateIndex + 1 + job.ModifyIndex = job.CreateIndex + 1 + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a deployment + d := mock.Deployment() + d.JobID = job.ID + d.JobCreateIndex = job.CreateIndex + d.JobModifyIndex = job.JobModifyIndex - 1 + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) } + + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is updated and had an old deployment, the scheduler properly cancels // the deployment. func TestServiceSched_CancelDeployment_NewerJob(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Generate a fake job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + h := NewHarness(t) - // Create a deployment for an old version of the job - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Generate a fake job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Upsert again to bump job version - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a deployment for an old version of the job + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Create a mock evaluation to kick the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } + // Upsert again to bump job version + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to kick the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) - } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) + } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } // Various table driven tests for carry forward // of past reschedule events func Test_updateRescheduleTracker(t *testing.T) { + t1 := time.Now().UTC() alloc := mock.Alloc() prevAlloc := mock.Alloc() @@ -4837,4 +4626,5 @@ func Test_updateRescheduleTracker(t *testing.T) { require.Equal(tc.expectedRescheduleEvents, alloc.RescheduleTracker.Events) }) } + } diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 76a0a69408c..639b2b8cf28 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -28,7 +28,7 @@ var BuiltinSchedulers = map[string]Factory{ // NewScheduler is used to instantiate and return a new scheduler // given the scheduler name, initial state, and planner. -func NewScheduler(name string, logger log.Logger, state State, planner Planner, allowPlanOptimization bool) (Scheduler, error) { +func NewScheduler(name string, logger log.Logger, state State, planner Planner) (Scheduler, error) { // Lookup the factory function factory, ok := BuiltinSchedulers[name] if !ok { @@ -36,12 +36,12 @@ func NewScheduler(name string, logger log.Logger, state State, planner Planner, } // Instantiate the scheduler - sched := factory(logger, state, planner, allowPlanOptimization) + sched := factory(logger, state, planner) return sched, nil } // Factory is used to instantiate a new Scheduler -type Factory func(log.Logger, State, Planner, bool) Scheduler +type Factory func(log.Logger, State, Planner) Scheduler // Scheduler is the top level instance for a scheduler. A scheduler is // meant to only encapsulate business logic, pushing the various plumbing diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index 2cefcba3478..5230ba0f71b 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -4,7 +4,7 @@ import ( "fmt" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" ) @@ -23,10 +23,6 @@ type SystemScheduler struct { state State planner Planner - // Temporary flag introduced till the code for sending/committing full allocs in the Plan can - // be safely removed - allowPlanOptimization bool - eval *structs.Evaluation job *structs.Job plan *structs.Plan @@ -45,12 +41,11 @@ type SystemScheduler struct { // NewSystemScheduler is a factory function to instantiate a new system // scheduler. -func NewSystemScheduler(logger log.Logger, state State, planner Planner, allowPlanOptimization bool) Scheduler { +func NewSystemScheduler(logger log.Logger, state State, planner Planner) Scheduler { return &SystemScheduler{ - logger: logger.Named("system_sched"), - state: state, - planner: planner, - allowPlanOptimization: allowPlanOptimization, + logger: logger.Named("system_sched"), + state: state, + planner: planner, } } @@ -115,7 +110,7 @@ func (s *SystemScheduler) process() (bool, error) { } // Create a plan - s.plan = s.eval.MakePlan(s.job, s.allowPlanOptimization) + s.plan = s.eval.MakePlan(s.job) // Reset the failed allocations s.failedTGAllocs = nil diff --git a/scheduler/system_sched_test.go b/scheduler/system_sched_test.go index 2509697de93..9461fd85eaa 100644 --- a/scheduler/system_sched_test.go +++ b/scheduler/system_sched_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hashicorp/go-memdb" + memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -15,1964 +15,1870 @@ import ( "github.com/stretchr/testify/require" ) -// COMPAT 0.11: Currently, all the tests run for 2 cases: -// 1) Allow plan optimization -// 2) Not allowing plan optimization -// The code for not allowing plan optimizations is in place to allow for safer cluster upgrades, -// and backwards compatibility with the existing raft logs. The older code will be removed later, -// and these tests should then no longer include the testing for case (2) - func TestSystemSched_JobRegister(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure no allocations are queued - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no allocations are queued + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure the plan allocated - plan := h.Plans[0] - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Get an allocation and mark it as failed - alloc := planned[4].Copy() - alloc.ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - if err := h1.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Ensure the plan allocated + plan := h.Plans[0] + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure we have created only one new allocation - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 1 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocation was placed on the same node as the older - // one - if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { - t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) - } - }) + // Get an allocation and mark it as failed + alloc := planned[4].Copy() + alloc.ClientStatus = structs.AllocClientStatusFailed + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State) + if err := h1.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure we have created only one new allocation + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 1 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocation was placed on the same node as the older + // one + if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { + t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) } } func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create another job with a lot of disk resource ask so that it doesn't fit - // the node - job1 := mock.SystemJob() - job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create another job with a lot of disk resource ask so that it doesn't fit + // the node + job1 := mock.SystemJob() + job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Create a new harness to test the scheduling result for the second job - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a new harness to test the scheduling result for the second job + h1 := NewHarnessWithState(t, h.State) + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) - noErr(t, err) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - }) + out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) + noErr(t, err) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } } func TestSystemSched_ExhaustResources(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create a service job which consumes most of the system resources - svcJob := mock.Job() - svcJob.TaskGroups[0].Count = 1 - svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: svcJob.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: svcJob.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create a service job which consumes most of the system resources + svcJob := mock.Job() + svcJob.TaskGroups[0].Count = 1 + svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: svcJob.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: svcJob.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a system job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Create a system job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // System scheduler will preempt the service job and would have placed eval1 - require := require.New(t) + // System scheduler will preempt the service job and would have placed eval1 + require := require.New(t) - newPlan := h.Plans[1] - require.Len(newPlan.NodeAllocation, 1) - require.Len(newPlan.NodePreemptions, 1) + newPlan := h.Plans[1] + require.Len(newPlan.NodeAllocation, 1) + require.Len(newPlan.NodePreemptions, 1) - for _, allocList := range newPlan.NodeAllocation { - require.Len(allocList, 1) - require.Equal(job.ID, allocList[0].JobID) - } + for _, allocList := range newPlan.NodeAllocation { + require.Len(allocList, 1) + require.Equal(job.ID, allocList[0].JobID) + } - for _, allocList := range newPlan.NodePreemptions { - require.Len(allocList, 1) - alloc, err := h.State.AllocByID(nil, allocList[0].ID) - noErr(t, err) - require.Equal(svcJob.ID, alloc.JobID) - } - // Ensure that we have no queued allocations on the second eval - queued := h.Evals[1].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } - }) + for _, allocList := range newPlan.NodePreemptions { + require.Len(allocList, 1) + require.Equal(svcJob.ID, allocList[0].JobID) + } + // Ensure that we have no queued allocations on the second eval + queued := h.Evals[1].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) } } func TestSystemSched_JobRegister_Annotate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - if i < 9 { - node.NodeClass = "foo" - } else { - node.NodeClass = "bar" - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job constraining on node class - job := mock.SystemJob() - fooConstraint := &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "foo", - Operand: "==", - } - job.Constraints = append(job.Constraints, fooConstraint) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + if i < 9 { + node.NodeClass = "foo" + } else { + node.NodeClass = "bar" + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job constraining on node class + job := mock.SystemJob() + fooConstraint := &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "foo", + Operand: "==", + } + job.Constraints = append(job.Constraints, fooConstraint) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 9 { - t.Fatalf("bad: %#v %d", planned, len(planned)) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 9 { + t.Fatalf("bad: %#v %d", planned, len(planned)) + } - // Ensure all allocations placed - if len(out) != 9 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Ensure all allocations placed + if len(out) != 9 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + h.AssertEvalStatus(t, structs.EvalStatusComplete) - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - expected := &structs.DesiredUpdates{Place: 9} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - } - }) + expected := &structs.DesiredUpdates{Place: 9} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) } } func TestSystemSched_JobRegister_AddNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a new node. - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a new node. + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan had no node updates - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Log(len(update)) - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated on the new node - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan had no node updates + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Log(len(update)) + t.Fatalf("bad: %#v", plan) + } - // Ensure it allocated on the right node - if _, ok := plan.NodeAllocation[node.ID]; !ok { - t.Fatalf("allocated on wrong node: %#v", plan) - } + // Ensure the plan allocated on the new node + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure it allocated on the right node + if _, ok := plan.NodeAllocation[node.ID]; !ok { + t.Fatalf("allocated on wrong node: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 11 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 11 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_AllocFail(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create NO nodes - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan as this should be a no-op. - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + h := NewHarness(t) + + // Create NO nodes + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no plan as this should be a no-op. + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = "my-job.web[0]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = "my-job.web[0]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify_Rolling(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - job2.Update = structs.UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 5, - } + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + job2.Update = structs.UpdateStrategy{ + Stagger: 30 * time.Second, + MaxParallel: 5, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a follow up eval was created - eval = h.Evals[0] - if eval.NextEval == "" { - t.Fatalf("missing next eval") - } + // Ensure a follow up eval was created + eval = h.Evals[0] + if eval.NextEval == "" { + t.Fatalf("missing next eval") + } - // Check for create - if len(h.CreateEvals) == 0 { - t.Fatalf("missing created eval") - } - create := h.CreateEvals[0] - if eval.NextEval != create.ID { - t.Fatalf("ID mismatch") - } - if create.PreviousEval != eval.ID { - t.Fatalf("missing previous eval") - } + // Check for create + if len(h.CreateEvals) == 0 { + t.Fatalf("missing created eval") + } + create := h.CreateEvals[0] + if eval.NextEval != create.ID { + t.Fatalf("ID mismatch") + } + if create.PreviousEval != eval.ID { + t.Fatalf("missing previous eval") + } - if create.TriggeredBy != structs.EvalTriggerRollingUpdate { - t.Fatalf("bad: %#v", create) - } - }) + if create.TriggeredBy != structs.EvalTriggerRollingUpdate { + t.Fatalf("bad: %#v", create) } } func TestSystemSched_JobModify_InPlace(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) - } - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) } - }) + } } } func TestSystemSched_JobDeregister_Purged(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobDeregister_Stopped(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Generate a fake job with allocations + job := mock.SystemJob() + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDown(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a down node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a down node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if p := planned[0]; p.DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && + p.ClientStatus != structs.AllocClientStatusLost { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain_Down(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure that the allocation is marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - expected := []string{alloc.ID} + // Ensure that the allocation is marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + expected := []string{alloc.ID} - if !reflect.DeepEqual(lostAllocs, expected) { - t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + if !reflect.DeepEqual(lostAllocs, expected) { + t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) } + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Log(len(planned)) - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Log(len(planned)) + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeUpdate(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + h := NewHarness(t) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_RetryLimit(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - h.Planner = &RejectPlan{h} - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) + h.Planner = &RejectPlan{h} - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) - }) + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) } + + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) } // This test ensures that the scheduler doesn't increment the queued allocation // count for a task group when allocations can't be created on currently // available nodes because of constrain mismatches. func TestSystemSched_Queued_With_Constraints(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register a node - node := mock.Node() - node.Attributes["kernel.name"] = "darwin" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a system job which can't be placed on the node - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register a node + node := mock.Node() + node.Attributes["kernel.name"] = "darwin" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a system job which can't be placed on the node + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } - }) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) } } func TestSystemSched_ChainedAlloc(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + h := NewHarness(t) - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) - job1 := mock.SystemJob() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Insert two more nodes - for i := 0; i < 2; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State) + job1 := mock.SystemJob() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Insert two more nodes + for i := 0; i < 2; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue - } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + plan := h1.Plans[0] - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue } - }) + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) + + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } + + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) } } func TestSystemSched_PlanWithDrainedNode(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.Drain = true - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create an allocation on each node - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - alloc.TaskGroup = "web" - - alloc2 := mock.Alloc() - alloc2.Job = job - alloc2.JobID = job.ID - alloc2.NodeID = node2.ID - alloc2.Name = "my-job.web2[0]" - alloc2.TaskGroup = "web2" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.Drain = true + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create an allocation on each node + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + alloc.TaskGroup = "web" + + alloc2 := mock.Alloc() + alloc2.Job = job + alloc2.JobID = job.ID + alloc2.NodeID = node2.ID + alloc2.Name = "my-job.web2[0]" + alloc2.TaskGroup = "web2" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan evicted the alloc on the failed node - planned := plan.NodeUpdate[node.ID] - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan didn't place - if len(plan.NodeAllocation) != 0 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted the alloc on the failed node + planned := plan.NodeUpdate[node.ID] + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } - // Ensure the allocations is stopped - if planned[0].DesiredDescription != allocNodeTainted { - t.Fatalf("bad: %#v", planned[0]) - } + // Ensure the plan didn't place + if len(plan.NodeAllocation) != 0 { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Ensure the allocations is stopped + if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad: %#v", planned[0]) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", - }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h := NewHarness(t) + + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - qa := h.Evals[0].QueuedAllocations - if qa["web"] != 0 || qa["web2"] != 0 { - t.Fatalf("bad queued allocations %#v", qa) - } + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + qa := h.Evals[0].QueuedAllocations + if qa["web"] != 0 || qa["web2"] != 0 { + t.Fatalf("bad queued allocations %#v", qa) } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_Preemption(t *testing.T) { - for _, allowPlanOptimization := range []bool{true, false} { - t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { - h := NewHarness(t, allowPlanOptimization) - - // Create nodes - var nodes []*structs.Node - for i := 0; i < 2; i++ { - node := mock.Node() - //TODO(preetha): remove in 0.11 - node.Resources = &structs.Resources{ - CPU: 3072, - MemoryMB: 5034, - DiskMB: 20 * 1024, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - node.NodeResources = &structs.NodeResources{ - Cpu: structs.NodeCpuResources{ - CpuShares: 3072, - }, - Memory: structs.NodeMemoryResources{ - MemoryMB: 5034, - }, - Disk: structs.NodeDiskResources{ - DiskMB: 20 * 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - nodes = append(nodes, node) - } - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, + h := NewHarness(t) + + // Create nodes + var nodes []*structs.Node + for i := 0; i < 2; i++ { + node := mock.Node() + //TODO(preetha): remove in 0.11 + node.Resources = &structs.Resources{ + CPU: 3072, + MemoryMB: 5034, + DiskMB: 20 * 1024, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, }, - }) - - // Create some low priority batch jobs and allocations for them - // One job uses a reserved port - job1 := mock.BatchJob() - job1.Type = structs.JobTypeBatch - job1.Priority = 20 - job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ + }, + } + node.NodeResources = &structs.NodeResources{ + Cpu: structs.NodeCpuResources{ + CpuShares: 3072, + }, + Memory: structs.NodeMemoryResources{ + MemoryMB: 5034, + }, + Disk: structs.NodeDiskResources{ + DiskMB: 20 * 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + nodes = append(nodes, node) + } + + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) + + // Create some low priority batch jobs and allocations for them + // One job uses a reserved port + job1 := mock.BatchJob() + job1.Type = structs.JobTypeBatch + job1.Priority = 20 + job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ + { + MBits: 200, + ReservedPorts: []structs.Port{ { - MBits: 200, - ReservedPorts: []structs.Port{ - { - Label: "web", - Value: 80, - }, - }, + Label: "web", + Value: 80, }, }, - } + }, + }, + } - alloc1 := mock.Alloc() - alloc1.Job = job1 - alloc1.JobID = job1.ID - alloc1.NodeID = nodes[0].ID - alloc1.Name = "my-job[0]" - alloc1.TaskGroup = job1.TaskGroups[0].Name - alloc1.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 200, - }, - }, - }, + alloc1 := mock.Alloc() + alloc1.Job = job1 + alloc1.JobID = job1.ID + alloc1.NodeID = nodes[0].ID + alloc1.Name = "my-job[0]" + alloc1.TaskGroup = job1.TaskGroups[0].Name + alloc1.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, }, - } - - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - job2 := mock.BatchJob() - job2.Type = structs.JobTypeBatch - job2.Priority = 20 - job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, Networks: []*structs.NetworkResource{ { - MBits: 200, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 200, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } - alloc2 := mock.Alloc() - alloc2.Job = job2 - alloc2.JobID = job2.ID - alloc2.NodeID = nodes[0].ID - alloc2.Name = "my-job[2]" - alloc2.TaskGroup = job2.TaskGroups[0].Name - alloc2.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - MBits: 200, - }, - }, - }, + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + job2 := mock.BatchJob() + job2.Type = structs.JobTypeBatch + job2.Priority = 20 + job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ + { + MBits: 200, + }, + }, + } + + alloc2 := mock.Alloc() + alloc2.Job = job2 + alloc2.JobID = job2.ID + alloc2.NodeID = nodes[0].ID + alloc2.Name = "my-job[2]" + alloc2.TaskGroup = job2.TaskGroups[0].Name + alloc2.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - job3 := mock.Job() - job3.Type = structs.JobTypeBatch - job3.Priority = 40 - job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, Networks: []*structs.NetworkResource{ { Device: "eth0", - MBits: 400, + IP: "192.168.0.100", + MBits: 200, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + job3 := mock.Job() + job3.Type = structs.JobTypeBatch + job3.Priority = 40 + job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + MBits: 400, + }, + }, + } - alloc3 := mock.Alloc() - alloc3.Job = job3 - alloc3.JobID = job3.ID - alloc3.NodeID = nodes[0].ID - alloc3.Name = "my-job[0]" - alloc3.TaskGroup = job3.TaskGroups[0].Name - alloc3.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 25, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 400, - }, - }, - }, + alloc3 := mock.Alloc() + alloc3.Job = job3 + alloc3.JobID = job3.ID + alloc3.NodeID = nodes[0].ID + alloc3.Name = "my-job[0]" + alloc3.TaskGroup = job3.TaskGroups[0].Name + alloc3.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 25, }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) - - // Create a high priority job and allocs for it - // These allocs should not be preempted - - job4 := mock.BatchJob() - job4.Type = structs.JobTypeBatch - job4.Priority = 100 - job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, Networks: []*structs.NetworkResource{ { - MBits: 100, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 400, }, }, - } + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) + + // Create a high priority job and allocs for it + // These allocs should not be preempted + + job4 := mock.BatchJob() + job4.Type = structs.JobTypeBatch + job4.Priority = 100 + job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, + Networks: []*structs.NetworkResource{ + { + MBits: 100, + }, + }, + } - alloc4 := mock.Alloc() - alloc4.Job = job4 - alloc4.JobID = job4.ID - alloc4.NodeID = nodes[0].ID - alloc4.Name = "my-job4[0]" - alloc4.TaskGroup = job4.TaskGroups[0].Name - alloc4.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2048, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 100, - }, - }, - }, + alloc4 := mock.Alloc() + alloc4.Job = job4 + alloc4.JobID = job4.ID + alloc4.NodeID = nodes[0].ID + alloc4.Name = "my-job4[0]" + alloc4.TaskGroup = job4.TaskGroups[0].Name + alloc4.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 2 * 1024, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 2048, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) - - // Create a system job such that it would need to preempt both allocs to succeed - job := mock.SystemJob() - job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1948, - MemoryMB: 256, Networks: []*structs.NetworkResource{ { - MBits: 800, - DynamicPorts: []structs.Port{{Label: "http"}}, + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 100, }, }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - require := require.New(t) - require.Nil(err) - - // Ensure a single plan - require.Equal(1, len(h.Plans)) - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - require.Nil(plan.Annotations) - - // Ensure the plan allocated on both nodes - var planned []*structs.Allocation - preemptingAllocId := "" - require.Equal(2, len(plan.NodeAllocation)) - - // The alloc that got placed on node 1 is the preemptor - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - for _, alloc := range allocList { - if alloc.NodeID == nodes[0].ID { - preemptingAllocId = alloc.ID - } - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 2 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) + + // Create a system job such that it would need to preempt both allocs to succeed + job := mock.SystemJob() + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1948, + MemoryMB: 256, + Networks: []*structs.NetworkResource{ + { + MBits: 800, + DynamicPorts: []structs.Port{{Label: "http"}}, + }, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Ensure all allocations placed - require.Equal(2, len(out)) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + require := require.New(t) + require.Nil(err) - // Verify that one node has preempted allocs - require.NotNil(plan.NodePreemptions[nodes[0].ID]) - preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + // Ensure a single plan + require.Equal(1, len(h.Plans)) + plan := h.Plans[0] - // Verify that three jobs have preempted allocs - require.Equal(3, len(preemptedAllocs)) + // Ensure the plan doesn't have annotations. + require.Nil(plan.Annotations) - expectedPreemptedAllocIDs := []string{alloc1.ID, alloc2.ID, alloc3.ID} + // Ensure the plan allocated on both nodes + var planned []*structs.Allocation + preemptingAllocId := "" + require.Equal(2, len(plan.NodeAllocation)) - // We expect job1, job2 and job3 to have preempted allocations - // job4 should not have any allocs preempted - for _, alloc := range preemptedAllocs { - require.Contains(expectedPreemptedAllocIDs, alloc.ID) - } - // Look up the preempted allocs by job ID - ws = memdb.NewWatchSet() - - for _, allocID := range expectedPreemptedAllocIDs { - evictedAlloc, err := h.State.AllocByID(ws, allocID) - noErr(t, err) - require.Equal(structs.AllocDesiredStatusEvict, evictedAlloc.DesiredStatus) - require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), evictedAlloc.DesiredDescription) + // The alloc that got placed on node 1 is the preemptor + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + for _, alloc := range allocList { + if alloc.NodeID == nodes[0].ID { + preemptingAllocId = alloc.ID } + } + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - }) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + require.Equal(2, len(out)) + + // Verify that one node has preempted allocs + require.NotNil(plan.NodePreemptions[nodes[0].ID]) + preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + + // Verify that three jobs have preempted allocs + require.Equal(3, len(preemptedAllocs)) + + expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} + + // We expect job1, job2 and job3 to have preempted allocations + // job4 should not have any allocs preempted + for _, alloc := range preemptedAllocs { + require.Contains(expectedPreemptedJobIDs, alloc.JobID) } + // Look up the preempted allocs by job ID + ws = memdb.NewWatchSet() + + for _, jobId := range expectedPreemptedJobIDs { + out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) + noErr(t, err) + for _, alloc := range out { + require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) + require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + } diff --git a/scheduler/testing.go b/scheduler/testing.go index 6eb43794572..876ff101db5 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/mitchellh/go-testing-interface" + testing "github.com/mitchellh/go-testing-interface" "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" @@ -54,29 +54,27 @@ type Harness struct { nextIndex uint64 nextIndexLock sync.Mutex - allowPlanOptimization bool + optimizePlan bool } // NewHarness is used to make a new testing harness -func NewHarness(t testing.T, allowPlanOptimization bool) *Harness { +func NewHarness(t testing.T) *Harness { state := state.TestStateStore(t) h := &Harness{ - t: t, - State: state, - nextIndex: 1, - allowPlanOptimization: allowPlanOptimization, + t: t, + State: state, + nextIndex: 1, } return h } // NewHarnessWithState creates a new harness with the given state for testing // purposes. -func NewHarnessWithState(t testing.T, state *state.StateStore, allowPlanOptimization bool) *Harness { +func NewHarnessWithState(t testing.T, state *state.StateStore) *Harness { return &Harness{ - t: t, - State: state, - nextIndex: 1, - allowPlanOptimization: allowPlanOptimization, + t: t, + State: state, + nextIndex: 1, } } @@ -137,7 +135,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er NodePreemptions: preemptedAllocs, } - if h.allowPlanOptimization { + if h.optimizePlan { req.AllocsStopped = allocsStopped req.AllocsUpdated = allocsUpdated } else { @@ -154,6 +152,12 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er return result, nil, err } +// OptimizePlan is a function used only for Harness to help set the optimzePlan field, +// since Harness doesn't have access to a Server object +func (h *Harness) OptimizePlan(optimize bool) { + h.optimizePlan = optimize +} + func updateCreateTimestamp(allocations []*structs.Allocation, now int64) { // Set the time the alloc was applied for the first time. This can be used // to approximate the scheduling time. @@ -234,15 +238,15 @@ func (h *Harness) Snapshot() State { // Scheduler is used to return a new scheduler from // a snapshot of current state using the harness for planning. -func (h *Harness) Scheduler(factory Factory, allowPlanOptimization bool) Scheduler { +func (h *Harness) Scheduler(factory Factory) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h, allowPlanOptimization) + return factory(logger, h.Snapshot(), h) } // Process is used to process an evaluation given a factory // function to create the scheduler func (h *Harness) Process(factory Factory, eval *structs.Evaluation) error { - sched := h.Scheduler(factory, h.allowPlanOptimization) + sched := h.Scheduler(factory) return sched.Process(eval) } diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 13864b43214..08f5812aac6 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -621,7 +621,7 @@ func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { } func TestSetStatus(t *testing.T) { - h := NewHarness(t, true) + h := NewHarness(t) logger := testlog.HCLogger(t) eval := mock.Eval() status := "a" @@ -640,7 +640,7 @@ func TestSetStatus(t *testing.T) { } // Test next evals - h = NewHarness(t, true) + h = NewHarness(t) next := mock.Eval() if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -656,7 +656,7 @@ func TestSetStatus(t *testing.T) { } // Test blocked evals - h = NewHarness(t, true) + h = NewHarness(t) blocked := mock.Eval() if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -672,7 +672,7 @@ func TestSetStatus(t *testing.T) { } // Test metrics - h = NewHarness(t, true) + h = NewHarness(t) metrics := map[string]*structs.AllocMetric{"foo": nil} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -688,7 +688,7 @@ func TestSetStatus(t *testing.T) { } // Test queued allocations - h = NewHarness(t, true) + h = NewHarness(t) queuedAllocs := map[string]int{"web": 1} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { @@ -704,7 +704,7 @@ func TestSetStatus(t *testing.T) { t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) } - h = NewHarness(t, true) + h = NewHarness(t) dID := uuid.Generate() if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { t.Fatalf("setStatus() failed: %v", err) From ee268a58db2aca9ddb737a5aba702446a8740e32 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Wed, 10 Apr 2019 17:15:04 -0700 Subject: [PATCH 5/7] Add comments to functions, and use require instead of assert --- nomad/plan_apply.go | 28 ++++++++++----- nomad/plan_apply_test.go | 42 ++++++++++++---------- nomad/plan_normalization_test.go | 9 +++-- nomad/state/state_store.go | 62 ++++++++++++++++++++++---------- nomad/state/state_store_test.go | 40 +++++++++------------ nomad/structs/structs.go | 29 +++++++++++---- nomad/util.go | 5 +-- scheduler/testing.go | 51 ++++++++++++++++++-------- 8 files changed, 170 insertions(+), 96 deletions(-) diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index 777b5e8cc6c..f9b8d7c63a2 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -157,7 +157,6 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap Deployment: result.Deployment, DeploymentUpdates: result.DeploymentUpdates, EvalID: plan.EvalID, - NodePreemptions: make([]*structs.Allocation, 0, len(result.NodePreemptions)), } preemptedJobIDs := make(map[structs.NamespacedID]struct{}) @@ -167,8 +166,9 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap // Initialize the allocs request using the new optimized log entry format. // Determine the minimum number of updates, could be more if there // are multiple updates per node - req.AllocsStopped = make([]*structs.Allocation, 0, len(result.NodeUpdate)) + req.AllocsStopped = make([]*structs.AllocationDiff, 0, len(result.NodeUpdate)) req.AllocsUpdated = make([]*structs.Allocation, 0, len(result.NodeAllocation)) + req.AllocsPreempted = make([]*structs.AllocationDiff, 0, len(result.NodePreemptions)) for _, updateList := range result.NodeUpdate { for _, stoppedAlloc := range updateList { @@ -186,7 +186,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap for _, preemptions := range result.NodePreemptions { for _, preemptedAlloc := range preemptions { - req.NodePreemptions = append(req.NodePreemptions, normalizePreemptedAlloc(preemptedAlloc, now)) + req.AllocsPreempted = append(req.AllocsPreempted, normalizePreemptedAlloc(preemptedAlloc, now)) // Gather jobids to create follow up evals appendNamespacedJobID(preemptedJobIDs, preemptedAlloc) @@ -201,8 +201,9 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap minUpdates := len(result.NodeUpdate) minUpdates += len(result.NodeAllocation) - // Initialize the allocs request using the older log entry format + // Initialize using the older log entry format for Alloc and NodePreemptions req.Alloc = make([]*structs.Allocation, 0, minUpdates) + req.NodePreemptions = make([]*structs.Allocation, 0, len(result.NodePreemptions)) for _, updateList := range result.NodeUpdate { req.Alloc = append(req.Alloc, updateList...) @@ -261,16 +262,24 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap return future, nil } -func normalizePreemptedAlloc(preemptedAlloc *structs.Allocation, now int64) *structs.Allocation { - return &structs.Allocation{ +// normalizePreemptedAlloc removes redundant fields from a preempted allocation and +// returns AllocationDiff. Since a preempted allocation is always an existing allocation, +// the struct returned by this method contains only the differential, which can be +// applied to an existing allocation, to yield the updated struct +func normalizePreemptedAlloc(preemptedAlloc *structs.Allocation, now int64) *structs.AllocationDiff { + return &structs.AllocationDiff{ ID: preemptedAlloc.ID, PreemptedByAllocation: preemptedAlloc.PreemptedByAllocation, ModifyTime: now, } } -func normalizeStoppedAlloc(stoppedAlloc *structs.Allocation, now int64) *structs.Allocation { - return &structs.Allocation{ +// normalizeStoppedAlloc removes redundant fields from a stopped allocation and +// returns AllocationDiff. Since a stopped allocation is always an existing allocation, +// the struct returned by this method contains only the differential, which can be +// applied to an existing allocation, to yield the updated struct +func normalizeStoppedAlloc(stoppedAlloc *structs.Allocation, now int64) *structs.AllocationDiff { + return &structs.AllocationDiff{ ID: stoppedAlloc.ID, DesiredDescription: stoppedAlloc.DesiredDescription, ClientStatus: stoppedAlloc.ClientStatus, @@ -278,6 +287,7 @@ func normalizeStoppedAlloc(stoppedAlloc *structs.Allocation, now int64) *structs } } +// appendNamespacedJobID appends the namespaced Job ID for the alloc to the jobIDs set func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *structs.Allocation) { id := structs.NamespacedID{Namespace: alloc.Namespace, ID: alloc.JobID} if _, ok := jobIDs[id]; !ok { @@ -285,6 +295,8 @@ func appendNamespacedJobID(jobIDs map[structs.NamespacedID]struct{}, alloc *stru } } +// updateAllocTimestamps sets the CreateTime and ModifyTime for the allocations +// to the timestamp provided func updateAllocTimestamps(allocations []*structs.Allocation, timestamp int64) { for _, alloc := range allocations { if alloc.CreateTime == 0 { diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index f58f95bc1cf..49733f19313 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -230,6 +230,8 @@ func TestPlanApply_applyPlan(t *testing.T) { assert.Equal(index, evalOut.ModifyIndex) } +// Verifies that applyPlan properly updates the constituent objects in MemDB, +// when the plan contains normalized allocs. func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { t.Parallel() s1 := TestServer(t, func(c *Config) { @@ -312,39 +314,41 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { EvalID: eval.ID, } + require := require.New(t) + assert := assert.New(t) + // Apply the plan future, err := s1.applyPlan(plan, planRes, snap) - assert := assert.New(t) - assert.Nil(err) + require.NoError(err) // Verify our optimistic snapshot is updated ws := memdb.NewWatchSet() allocOut, err := snap.AllocByID(ws, alloc.ID) - assert.Nil(err) - assert.NotNil(allocOut) + require.NoError(err) + require.NotNil(allocOut) deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) - assert.Nil(err) - assert.NotNil(deploymentOut) + require.NoError(err) + require.NotNil(deploymentOut) // Check plan does apply cleanly index, err := planWaitFuture(future) - assert.Nil(err) + require.NoError(err) assert.NotEqual(0, index) // Lookup the allocation fsmState := s1.fsm.State() allocOut, err = fsmState.AllocByID(ws, alloc.ID) - assert.Nil(err) - assert.NotNil(allocOut) + require.NoError(err) + require.NotNil(allocOut) assert.True(allocOut.CreateTime > 0) assert.True(allocOut.ModifyTime > 0) assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) // Verify stopped alloc diff applied cleanly updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) - assert.Nil(err) - assert.NotNil(updatedStoppedAlloc) + require.NoError(err) + require.NotNil(updatedStoppedAlloc) assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) @@ -352,8 +356,8 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { // Verify preempted alloc diff applied cleanly updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) - assert.Nil(err) - assert.NotNil(updatedPreemptedAlloc) + require.NoError(err) + require.NotNil(updatedPreemptedAlloc) assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) assert.Equal(updatedPreemptedAlloc.DesiredDescription, "Preempted by alloc ID "+preemptedAllocDiff.PreemptedByAllocation) @@ -361,20 +365,20 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { // Lookup the new deployment dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) - assert.Nil(err) - assert.NotNil(dout) + require.NoError(err) + require.NotNil(dout) // Lookup the updated deployment dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) - assert.Nil(err) - assert.NotNil(dout2) + require.NoError(err) + require.NotNil(dout2) assert.Equal(desiredStatus, dout2.Status) assert.Equal(desiredStatusDescription, dout2.StatusDescription) // Lookup updated eval evalOut, err := fsmState.EvalByID(ws, eval.ID) - assert.Nil(err) - assert.NotNil(evalOut) + require.NoError(err) + require.NotNil(evalOut) assert.Equal(index, evalOut.ModifyIndex) } diff --git a/nomad/plan_normalization_test.go b/nomad/plan_normalization_test.go index ab5f5b5caa5..0cd9a4d6eb1 100644 --- a/nomad/plan_normalization_test.go +++ b/nomad/plan_normalization_test.go @@ -11,6 +11,9 @@ import ( "github.com/ugorji/go/codec" ) +// This test compares the size of the normalized + OmitEmpty raft plan log entry +// with the earlier denormalized log. +// // Whenever this test is changed, care should be taken to ensure the older msgpack size // is recalculated when new fields are introduced in ApplyPlanResultsRequest func TestPlanNormalize(t *testing.T) { @@ -33,12 +36,12 @@ func TestPlanNormalize(t *testing.T) { } now := time.Now().UTC().UnixNano() - mockStoppedAllocSlice := make([]*structs.Allocation, numStoppedAllocs) + mockStoppedAllocSlice := make([]*structs.AllocationDiff, numStoppedAllocs) for i := 0; i < numStoppedAllocs; i++ { mockStoppedAllocSlice = append(mockStoppedAllocSlice, normalizeStoppedAlloc(mockAlloc, now)) } - mockPreemptionAllocSlice := make([]*structs.Allocation, numPreemptedAllocs) + mockPreemptionAllocSlice := make([]*structs.AllocationDiff, numPreemptedAllocs) for i := 0; i < numPreemptedAllocs; i++ { mockPreemptionAllocSlice = append(mockPreemptionAllocSlice, normalizePreemptedAlloc(mockAlloc, now)) } @@ -49,7 +52,7 @@ func TestPlanNormalize(t *testing.T) { AllocsUpdated: mockUpdatedAllocSlice, AllocsStopped: mockStoppedAllocSlice, }, - NodePreemptions: mockPreemptionAllocSlice, + AllocsPreempted: mockPreemptionAllocSlice, } handle := structs.MsgpackHandle diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 24b90a2f45b..84b58fdfdf9 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -175,12 +175,18 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR return err } - err = snapshot.DenormalizeAllocationsSlice(results.AllocsStopped, results.Job) + allocsStopped, err := snapshot.DenormalizeAllocationDiffSlice(results.AllocsStopped, results.Job) if err != nil { return err } - err = snapshot.DenormalizeAllocationsSlice(results.NodePreemptions, results.Job) + allocsPreempted, err := snapshot.DenormalizeAllocationDiffSlice(results.AllocsPreempted, results.Job) + if err != nil { + return err + } + + // COMPAT 0.11: Remove this denormalization when NodePreemptions is removed + results.NodePreemptions, err = snapshot.DenormalizeAllocationSlice(results.NodePreemptions, results.Job) if err != nil { return err } @@ -210,31 +216,31 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR } } - noOfAllocs := len(results.NodePreemptions) - - if len(results.Alloc) > 0 { + numAllocs := 0 + if len(results.Alloc) > 0 || len(results.NodePreemptions) > 0 { // COMPAT 0.11: This branch will be removed, when Alloc is removed // Attach the job to all the allocations. It is pulled out in the payload to // avoid the redundancy of encoding, but should be denormalized prior to // being inserted into MemDB. addComputedAllocAttrs(results.Alloc, results.Job) - noOfAllocs += len(results.Alloc) + numAllocs = len(results.Alloc) + len(results.NodePreemptions) } else { // Attach the job to all the allocations. It is pulled out in the payload to // avoid the redundancy of encoding, but should be denormalized prior to // being inserted into MemDB. addComputedAllocAttrs(results.AllocsUpdated, results.Job) - noOfAllocs += len(results.AllocsStopped) + len(results.AllocsUpdated) + numAllocs = len(allocsStopped) + len(results.AllocsUpdated) + len(allocsPreempted) } - allocsToUpsert := make([]*structs.Allocation, 0, noOfAllocs) + allocsToUpsert := make([]*structs.Allocation, 0, numAllocs) - // COMPAT 0.11: This append should be removed when Alloc is removed + // COMPAT 0.11: Both these appends should be removed when Alloc and NodePreemptions are removed allocsToUpsert = append(allocsToUpsert, results.Alloc...) + allocsToUpsert = append(allocsToUpsert, results.NodePreemptions...) - allocsToUpsert = append(allocsToUpsert, results.AllocsStopped...) + allocsToUpsert = append(allocsToUpsert, allocsStopped...) allocsToUpsert = append(allocsToUpsert, results.AllocsUpdated...) - allocsToUpsert = append(allocsToUpsert, results.NodePreemptions...) + allocsToUpsert = append(allocsToUpsert, allocsPreempted...) if err := s.upsertAllocsImpl(index, allocsToUpsert, txn); err != nil { return err @@ -251,6 +257,8 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR return nil } +// addComputedAllocAttrs adds the computed/derived attributes to the allocation. +// This method is used when an allocation is being denormalized. func addComputedAllocAttrs(allocs []*structs.Allocation, job *structs.Job) { structs.DenormalizeAllocationJobs(job, allocs) @@ -4111,24 +4119,40 @@ type StateSnapshot struct { // Allocation for each of the Allocation diffs and merges the updated attributes with // the existing Allocation, and attaches the Job provided func (s *StateSnapshot) DenormalizeAllocationsMap(nodeAllocations map[string][]*structs.Allocation, job *structs.Job) error { - for _, allocDiffs := range nodeAllocations { - if err := s.DenormalizeAllocationsSlice(allocDiffs, job); err != nil { + for nodeID, allocs := range nodeAllocations { + denormalizedAllocs, err := s.DenormalizeAllocationSlice(allocs, job) + if err != nil { return err } + + nodeAllocations[nodeID] = denormalizedAllocs } return nil } -// DenormalizeAllocationsSlice queries the Allocation for each of the Allocation diffs and merges +// DenormalizeAllocationSlice queries the Allocation for each allocation diff +// represented as an Allocation and merges the updated attributes with the existing +// Allocation, and attaches the Job provided. +func (s *StateSnapshot) DenormalizeAllocationSlice(allocs []*structs.Allocation, job *structs.Job) ([]*structs.Allocation, error) { + allocDiffs := make([]*structs.AllocationDiff, len(allocs)) + for i, alloc := range allocs { + allocDiffs[i] = alloc.AllocationDiff() + } + + return s.DenormalizeAllocationDiffSlice(allocDiffs, job) +} + +// DenormalizeAllocationDiffSlice queries the Allocation for each AllocationDiff and merges // the updated attributes with the existing Allocation, and attaches the Job provided -func (s *StateSnapshot) DenormalizeAllocationsSlice(allocDiffs []*structs.Allocation, job *structs.Job) error { +func (s *StateSnapshot) DenormalizeAllocationDiffSlice(allocDiffs []*structs.AllocationDiff, job *structs.Job) ([]*structs.Allocation, error) { // Output index for denormalized Allocations j := 0 + denormalizedAllocs := make([]*structs.Allocation, len(allocDiffs)) for _, allocDiff := range allocDiffs { alloc, err := s.AllocByID(nil, allocDiff.ID) if err != nil { - return fmt.Errorf("alloc lookup failed: %v", err) + return nil, fmt.Errorf("alloc lookup failed: %v", err) } if alloc == nil { continue @@ -4156,12 +4180,12 @@ func (s *StateSnapshot) DenormalizeAllocationsSlice(allocDiffs []*structs.Alloca } // Update the allocDiff in the slice to equal the denormalized alloc - allocDiffs[j] = allocCopy + denormalizedAllocs[j] = allocCopy j++ } // Retain only the denormalized Allocations in the slice - allocDiffs = allocDiffs[:j] - return nil + denormalizedAllocs = denormalizedAllocs[:j] + return denormalizedAllocs, nil } func getPreemptedAllocDesiredDescription(PreemptedByAllocID string) string { diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 0e0539b48d7..8b6bf65b590 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -152,56 +152,50 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { stoppedAlloc := mock.Alloc() stoppedAlloc.Job = job - stoppedAllocDiff := &structs.Allocation{ + stoppedAllocDiff := &structs.AllocationDiff{ ID: stoppedAlloc.ID, DesiredDescription: "desired desc", ClientStatus: structs.AllocClientStatusLost, } preemptedAlloc := mock.Alloc() preemptedAlloc.Job = job - preemptedAllocDiff := &structs.Allocation{ + preemptedAllocDiff := &structs.AllocationDiff{ ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } - if err := state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc}); err != nil { - t.Fatalf("err: %v", err) - } - - if err := state.UpsertJob(999, job); err != nil { - t.Fatalf("err: %v", err) - } + require := require.New(t) + require.NoError(state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc})) + require.NoError(state.UpsertJob(999, job)) eval := mock.Eval() eval.JobID = job.ID // Create an eval - if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(state.UpsertEvals(1, []*structs.Evaluation{eval})) // Create a plan result res := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ AllocsUpdated: []*structs.Allocation{alloc}, - AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, + AllocsStopped: []*structs.AllocationDiff{stoppedAllocDiff}, Job: job, }, EvalID: eval.ID, - NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, + AllocsPreempted: []*structs.AllocationDiff{preemptedAllocDiff}, } assert := assert.New(t) planModifyIndex := uint64(1000) err := state.UpsertPlanResults(planModifyIndex, &res) - assert.Nil(err) + require.NoError(err) ws := memdb.NewWatchSet() out, err := state.AllocByID(ws, alloc.ID) - assert.Nil(err) + require.NoError(err) assert.Equal(alloc, out) updatedStoppedAlloc, err := state.AllocByID(ws, stoppedAlloc.ID) - assert.Nil(err) + require.NoError(err) assert.Equal(stoppedAllocDiff.DesiredDescription, updatedStoppedAlloc.DesiredDescription) assert.Equal(structs.AllocDesiredStatusStop, updatedStoppedAlloc.DesiredStatus) assert.Equal(stoppedAllocDiff.ClientStatus, updatedStoppedAlloc.ClientStatus) @@ -209,23 +203,21 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) updatedPreemptedAlloc, err := state.AllocByID(ws, preemptedAlloc.ID) - assert.Nil(err) + require.NoError(err) assert.Equal(structs.AllocDesiredStatusEvict, updatedPreemptedAlloc.DesiredStatus) assert.Equal(preemptedAllocDiff.PreemptedByAllocation, updatedPreemptedAlloc.PreemptedByAllocation) assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) index, err := state.Index("allocs") - assert.Nil(err) + require.NoError(err) assert.EqualValues(planModifyIndex, index) - if watchFired(ws) { - t.Fatalf("bad") - } + require.False(watchFired(ws)) evalOut, err := state.EvalByID(ws, eval.ID) - assert.Nil(err) - assert.NotNil(evalOut) + require.NoError(err) + require.NotNil(evalOut) assert.EqualValues(planModifyIndex, evalOut.ModifyIndex) } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d18d6b9c54f..8d2ae8a7f2c 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -660,9 +660,15 @@ type ApplyPlanResultsRequest struct { // the evaluation itself being updated. EvalID string - // NodePreemptions is a slice of allocation diffs from other lower priority jobs + // COMPAT 0.11 + // NodePreemptions is a slice of allocations from other lower priority jobs + // that are preempted. Preempted allocations are marked as evicted. + // Deprecated: Replaced with AllocsPreempted which contains only the diff + NodePreemptions []*Allocation + + // AllocsPreempted is a slice of allocation diffs from other lower priority jobs // that are preempted. Preempted allocations are marked as evicted. - NodePreemptions []*AllocationDiff + AllocsPreempted []*AllocationDiff // PreemptionEvals is a slice of follow up evals for jobs whose allocations // have been preempted to place allocs in this plan @@ -7294,10 +7300,6 @@ type Allocation struct { ModifyTime int64 } -// AllocationDiff is a type alias for Allocation used to indicate that a diff is -// and not the entire allocation -type AllocationDiff = Allocation - // Index returns the index of the allocation. If the allocation is from a task // group with count greater than 1, there will be multiple allocations for it. func (a *Allocation) Index() uint { @@ -7687,6 +7689,19 @@ func (a *Allocation) Stub() *AllocListStub { } } +// AllocationDiff converts an Allocation type to an AllocationDiff type +// If at any time, modification are made to AllocationDiff so that an +// Allocation can no longer be safely converted to AllocationDiff, +// this method should be changed accordingly. +func (a *Allocation) AllocationDiff() *AllocationDiff { + return (*AllocationDiff)(a) +} + +// AllocationDiff is another named type for Allocation (to use the same fields), +// which is used to represent the delta for an Allocation. If you need a method +// defined on the al +type AllocationDiff Allocation + // AllocListStub is used to return a subset of alloc information type AllocListStub struct { ID string @@ -8468,6 +8483,8 @@ func (p *Plan) IsNoOp() bool { len(p.DeploymentUpdates) == 0 } +// NormalizeAllocations normalizes allocations to remove fields that can +// be fetched from the MemDB instead of sending over the wire func (p *Plan) NormalizeAllocations() { for _, allocs := range p.NodeUpdate { for i, alloc := range allocs { diff --git a/nomad/util.go b/nomad/util.go index 74dc41e04ec..d4d8e0a20e7 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -148,8 +148,9 @@ func isNomadServer(m serf.Member) (bool, *serverParts) { return true, parts } -// ServersMeetMinimumVersion returns whether the given alive servers are at least on the -// given Nomad version +// ServersMeetMinimumVersion returns whether the Nomad servers are at least on the +// given Nomad version. The checkFailedServers parameter specifies whether version +// for the failed servers should be verified. func ServersMeetMinimumVersion(members []serf.Member, minVersion *version.Version, checkFailedServers bool) bool { for _, member := range members { if valid, parts := isNomadServer(member); valid && (parts.Status == serf.StatusAlive || (checkFailedServers && parts.Status == serf.StatusFailed)) { diff --git a/scheduler/testing.go b/scheduler/testing.go index 876ff101db5..cb6059c5469 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -104,10 +104,6 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er // Flatten evicts and allocs now := time.Now().UTC().UnixNano() - allocsStopped := make([]*structs.Allocation, 0, len(result.NodeUpdate)) - for _, updateList := range plan.NodeUpdate { - allocsStopped = append(allocsStopped, updateList...) - } allocsUpdated := make([]*structs.Allocation, 0, len(result.NodeAllocation)) for _, allocList := range plan.NodeAllocation { @@ -115,15 +111,6 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er } updateCreateTimestamp(allocsUpdated, now) - // Set modify time for preempted allocs and flatten them - var preemptedAllocs []*structs.Allocation - for _, preemptions := range result.NodePreemptions { - for _, alloc := range preemptions { - alloc.ModifyTime = now - preemptedAllocs = append(preemptedAllocs, alloc) - } - } - // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ @@ -132,19 +119,53 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er Deployment: plan.Deployment, DeploymentUpdates: plan.DeploymentUpdates, EvalID: plan.EvalID, - NodePreemptions: preemptedAllocs, } if h.optimizePlan { - req.AllocsStopped = allocsStopped + stoppedAllocDiffs := make([]*structs.AllocationDiff, 0, len(result.NodeUpdate)) + for _, updateList := range plan.NodeUpdate { + for _, stoppedAlloc := range updateList { + stoppedAllocDiffs = append(stoppedAllocDiffs, stoppedAlloc.AllocationDiff()) + } + } + req.AllocsStopped = stoppedAllocDiffs + req.AllocsUpdated = allocsUpdated + + preemptedAllocDiffs := make([]*structs.AllocationDiff, 0, len(result.NodePreemptions)) + for _, preemptions := range plan.NodePreemptions { + for _, preemptedAlloc := range preemptions { + allocDiff := preemptedAlloc.AllocationDiff() + allocDiff.ModifyTime = now + preemptedAllocDiffs = append(preemptedAllocDiffs, allocDiff) + } + } + req.AllocsPreempted = preemptedAllocDiffs } else { // COMPAT 0.11: Handles unoptimized log format var allocs []*structs.Allocation + + allocsStopped := make([]*structs.Allocation, 0, len(result.NodeUpdate)) + for _, updateList := range plan.NodeUpdate { + allocsStopped = append(allocsStopped, updateList...) + } allocs = append(allocs, allocsStopped...) + allocs = append(allocs, allocsUpdated...) updateCreateTimestamp(allocs, now) + req.Alloc = allocs + + // Set modify time for preempted allocs and flatten them + var preemptedAllocs []*structs.Allocation + for _, preemptions := range result.NodePreemptions { + for _, alloc := range preemptions { + alloc.ModifyTime = now + preemptedAllocs = append(preemptedAllocs, alloc) + } + } + + req.NodePreemptions = preemptedAllocs } // Apply the full plan From ab2718c8505a26913e1b0d8e7ac90ce604bb18b2 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Wed, 24 Apr 2019 11:01:13 -0700 Subject: [PATCH 6/7] Return error when preempted/stopped alloc doesn't exist during denormalization --- nomad/state/state_store.go | 2 +- nomad/state/state_store_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 84b58fdfdf9..276c6920d29 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -4155,7 +4155,7 @@ func (s *StateSnapshot) DenormalizeAllocationDiffSlice(allocDiffs []*structs.All return nil, fmt.Errorf("alloc lookup failed: %v", err) } if alloc == nil { - continue + return nil, fmt.Errorf("alloc %v doesn't exist", allocDiff.ID) } // Merge the updates to the Allocation diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 8b6bf65b590..013f9cab932 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -7054,6 +7054,31 @@ func TestStateStore_Abandon(t *testing.T) { } } +// Verifies that an error is returned when an allocation doesn't exist in the state store. +func TestStateSnapshot_DenormalizeAllocationDiffSlice_AllocDoesNotExist(t *testing.T) { + state := testStateStore(t) + alloc := mock.Alloc() + require := require.New(t) + + // Insert job + err := state.UpsertJob(999, alloc.Job) + require.NoError(err) + + allocDiffs := []*structs.AllocationDiff{ + { + ID: alloc.ID, + }, + } + + snap, err := state.Snapshot() + require.NoError(err) + + denormalizedAllocs, err := snap.DenormalizeAllocationDiffSlice(allocDiffs, alloc.Job) + + require.EqualError(err, fmt.Sprintf("alloc %v doesn't exist", alloc.ID)) + require.Nil(denormalizedAllocs) +} + // watchFired is a helper for unit tests that returns if the given watch set // fired (it doesn't care which watch actually fired). This uses a fixed // timeout since we already expect the event happened before calling this and From 23bc1f26800d25e0dc37b50e9923cd8bbe8a42b7 Mon Sep 17 00:00:00 2001 From: Arshneet Singh Date: Wed, 24 Apr 2019 11:01:59 -0700 Subject: [PATCH 7/7] Change min version required for plan optimization --- nomad/plan_apply_test.go | 2 +- nomad/util.go | 2 +- nomad/worker_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 49733f19313..e623509842a 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -235,7 +235,7 @@ func TestPlanApply_applyPlan(t *testing.T) { func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { t.Parallel() s1 := TestServer(t, func(c *Config) { - c.Build = "0.9.1" + c.Build = "0.9.2" }) defer s1.Shutdown() testutil.WaitForLeader(t, s1.RPC) diff --git a/nomad/util.go b/nomad/util.go index d4d8e0a20e7..ccd4504af05 100644 --- a/nomad/util.go +++ b/nomad/util.go @@ -17,7 +17,7 @@ import ( // MinVersionPlanNormalization is the minimum version to support the // normalization of Plan in SubmitPlan, and the denormalization raft log entry committed // in ApplyPlanResultsRequest -var MinVersionPlanNormalization = version.Must(version.NewVersion("0.9.1")) +var MinVersionPlanNormalization = version.Must(version.NewVersion("0.9.2")) // ensurePath is used to make sure a path exists func ensurePath(path string, dir bool) error { diff --git a/nomad/worker_test.go b/nomad/worker_test.go index a03e739bfc3..4bc8628aff6 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -396,7 +396,7 @@ func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { s1 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.EnabledSchedulers = []string{structs.JobTypeService} - c.Build = "0.9.1" + c.Build = "0.9.2" }) defer s1.Shutdown() testutil.WaitForLeader(t, s1.RPC)