From 242cc191a19c6e934712394805753cbe5694dd98 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Tue, 8 May 2018 17:26:36 -0500 Subject: [PATCH 01/14] Work in progress - force rescheduling of failed allocs --- command/agent/job_endpoint.go | 8 ++++++- nomad/job_endpoint.go | 39 ++++++++++++++++++++++++++++++----- nomad/structs/structs.go | 33 ++++++++++++++++++++++++++--- scheduler/reconcile_util.go | 5 +++++ 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 9ff26acc66d..f69f763c82d 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -90,8 +90,14 @@ func (s *HTTPServer) jobForceEvaluate(resp http.ResponseWriter, req *http.Reques if req.Method != "PUT" && req.Method != "POST" { return nil, CodedError(405, ErrInvalidMethod) } + + evalOptions := structs.EvalOptions{} + if _, ok := req.URL.Query()["force"]; ok { + evalOptions.ForceReschedule = true + } args := structs.JobEvaluateRequest{ - JobID: jobName, + JobID: jobName, + EvalOptions: evalOptions, } s.parseWriteRequest(req, &args.WriteRequest) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 4acc885fffa..18e03d582a4 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -39,6 +39,13 @@ var ( RTarget: ">= 0.6.1", Operand: structs.ConstraintVersion, } + + // allowRescheduleTransition is the transition that allows failed + // allocations to be force rescheduled. We create a one off + // variable to avoid creating a new object for every request. + allowForceRescheduleTransition = &structs.DesiredTransition{ + ForceReschedule: helper.BoolToPtr(true), + } ) // Job endpoint is used for job interactions @@ -538,6 +545,27 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis return fmt.Errorf("can't evaluate parameterized job") } + forceRescheduleAllocs := make(map[string]*structs.DesiredTransition) + if args.EvalOptions.ForceReschedule { + // Find any failed allocs that could be force rescheduled + allocs, err := snap.AllocsByJob(ws, args.RequestNamespace(), args.JobID, false) + if err != nil { + return err + } + + for _, alloc := range allocs { + taskGroup := job.LookupTaskGroup(alloc.TaskGroup) + // Forcing rescheduling is only allowed if task group has rescheduling enabled + if taskGroup != nil && taskGroup.ReschedulePolicy != nil && taskGroup.ReschedulePolicy.Enabled() { + continue + } + + if alloc.NextAllocation == "" && alloc.ClientStatus == structs.AllocClientStatusFailed { + forceRescheduleAllocs[alloc.ID] = allowForceRescheduleTransition + } + } + } + // Create a new evaluation eval := &structs.Evaluation{ ID: uuid.Generate(), @@ -549,13 +577,14 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis JobModifyIndex: job.ModifyIndex, Status: structs.EvalStatusPending, } - update := &structs.EvalUpdateRequest{ - Evals: []*structs.Evaluation{eval}, - WriteRequest: structs.WriteRequest{Region: args.Region}, + + // Create a AllocUpdateDesiredTransitionRequest request with the eval and any forced rescheduled allocs + updateTransitionReq := &structs.AllocUpdateDesiredTransitionRequest{ + Allocs: forceRescheduleAllocs, + Evals: []*structs.Evaluation{eval}, } + _, evalIndex, err := j.srv.raftApply(structs.AllocUpdateDesiredTransitionRequestType, updateTransitionReq) - // Commit this evaluation via Raft - _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) if err != nil { j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err) return err diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 70fbc05546f..c1d6df810fb 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -465,10 +465,15 @@ type JobDeregisterOptions struct { // JobEvaluateRequest is used when we just need to re-evaluate a target job type JobEvaluateRequest struct { - JobID string + JobID string + EvalOptions EvalOptions WriteRequest } +type EvalOptions struct { + ForceReschedule bool +} + // JobSpecificRequest is used when we just need to specify a target job type JobSpecificRequest struct { JobID string @@ -2988,13 +2993,17 @@ func (r *ReschedulePolicy) Copy() *ReschedulePolicy { return nrp } +func (r *ReschedulePolicy) Enabled() bool { + enabled := r != nil && (r.Attempts > 0 || r.Unlimited) + return enabled +} + // Validate uses different criteria to validate the reschedule policy // Delay must be a minimum of 5 seconds // Delay Ceiling is ignored if Delay Function is "constant" // Number of possible attempts is validated, given the interval, delay and delay function func (r *ReschedulePolicy) Validate() error { - enabled := r != nil && (r.Attempts > 0 || r.Unlimited) - if !enabled { + if !r.Enabled() { return nil } var mErr multierror.Error @@ -5608,6 +5617,11 @@ type DesiredTransition struct { // automatically eligible. An example is an allocation that is part of a // deployment. Reschedule *bool + + // ForceReschedule is used to indicate that this allocation must be rescheduled. + // This field is only used when operators want to force a placement even if + // a failed allocation is not eligible to be rescheduled + ForceReschedule *bool } // Merge merges the two desired transitions, preferring the values from the @@ -5620,6 +5634,10 @@ func (d *DesiredTransition) Merge(o *DesiredTransition) { if o.Reschedule != nil { d.Reschedule = o.Reschedule } + + if o.ForceReschedule != nil { + d.ForceReschedule = o.ForceReschedule + } } // ShouldMigrate returns whether the transition object dictates a migration. @@ -5633,6 +5651,15 @@ func (d *DesiredTransition) ShouldReschedule() bool { return d.Reschedule != nil && *d.Reschedule } +// ShouldForceReschedule returns whether the transition object dictates a +// forced rescheduling. +func (d *DesiredTransition) ShouldForceReschedule() bool { + if d == nil { + return false + } + return d.ForceReschedule != nil && *d.ForceReschedule +} + const ( AllocDesiredStatusRun = "run" // Allocation should run AllocDesiredStatusStop = "stop" // Allocation should stop diff --git a/scheduler/reconcile_util.go b/scheduler/reconcile_util.go index b59fd8209cc..87fa4f93644 100644 --- a/scheduler/reconcile_util.go +++ b/scheduler/reconcile_util.go @@ -327,6 +327,11 @@ func updateByReschedulable(alloc *structs.Allocation, now time.Time, evalID stri return } + // Check if the allocation is marked as it should be force rescheduled + if alloc.DesiredTransition.ShouldForceReschedule() { + rescheduleNow = true + } + // Reschedule if the eval ID matches the alloc's followup evalID or if its close to its reschedule time rescheduleTime, eligible := alloc.NextRescheduleTime() if eligible && (alloc.FollowupEvalID == evalID || rescheduleTime.Sub(now) <= rescheduleWindowSize) { From 3b7d23f364b6416877539022f89adb4855d2a70a Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Tue, 8 May 2018 20:00:06 -0500 Subject: [PATCH 02/14] Fix logic inversion in force rescheduling --- nomad/job_endpoint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 18e03d582a4..c6c12106932 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -546,6 +546,7 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis } forceRescheduleAllocs := make(map[string]*structs.DesiredTransition) + if args.EvalOptions.ForceReschedule { // Find any failed allocs that could be force rescheduled allocs, err := snap.AllocsByJob(ws, args.RequestNamespace(), args.JobID, false) @@ -556,7 +557,7 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis for _, alloc := range allocs { taskGroup := job.LookupTaskGroup(alloc.TaskGroup) // Forcing rescheduling is only allowed if task group has rescheduling enabled - if taskGroup != nil && taskGroup.ReschedulePolicy != nil && taskGroup.ReschedulePolicy.Enabled() { + if taskGroup == nil || taskGroup.ReschedulePolicy == nil || !taskGroup.ReschedulePolicy.Enabled() { continue } From 268a99e71a49cc32572585e5833d4bf70b289bef Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 9 May 2018 11:30:42 -0500 Subject: [PATCH 03/14] Add unit tests for forced rescheduling --- nomad/job_endpoint_test.go | 74 +++++++++++++++++++++++++++++++++++++ scheduler/reconcile_test.go | 72 ++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 92067040ad9..85197ab8043 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -1297,6 +1297,80 @@ func TestJobEndpoint_Evaluate(t *testing.T) { } } +func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { + require := require.New(t) + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + job := mock.Job() + req := &structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Fetch the response + var resp structs.JobRegisterResponse + err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) + require.Nil(err) + require.NotEqual(0, resp.Index) + + state := s1.fsm.State() + job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID) + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.TaskGroup = job.TaskGroups[0].Name + alloc.Namespace = job.Namespace + alloc.ClientStatus = structs.AllocClientStatusFailed + err = s1.State().UpsertAllocs(resp.Index+1, []*structs.Allocation{alloc}) + require.Nil(err) + + // Force a re-evaluation + reEval := &structs.JobEvaluateRequest{ + JobID: job.ID, + EvalOptions: structs.EvalOptions{ForceReschedule: true}, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Fetch the response + err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) + require.Nil(err) + require.NotEqual(0, resp.Index) + + // Lookup the evaluation + ws := memdb.NewWatchSet() + eval, err := state.EvalByID(ws, resp.EvalID) + require.Nil(err) + require.NotNil(eval) + require.Equal(eval.CreateIndex, resp.EvalCreateIndex) + require.Equal(eval.Priority, job.Priority) + require.Equal(eval.Type, job.Type) + require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) + require.Equal(eval.JobID, job.ID) + require.Equal(eval.JobModifyIndex, resp.JobModifyIndex) + require.Equal(eval.Status, structs.EvalStatusPending) + + // Lookup the alloc, verify DesiredTransition ForceReschedule + alloc, err = state.AllocByID(ws, alloc.ID) + require.NotNil(alloc) + require.Nil(err) + require.True(*alloc.DesiredTransition.ForceReschedule) +} + func TestJobEndpoint_Evaluate_ACL(t *testing.T) { t.Parallel() require := require.New(t) diff --git a/scheduler/reconcile_test.go b/scheduler/reconcile_test.go index 9d22439e2ff..8ac0db02020 100644 --- a/scheduler/reconcile_test.go +++ b/scheduler/reconcile_test.go @@ -4570,3 +4570,75 @@ func TestReconciler_SuccessfulDeploymentWithFailedAllocs_Reschedule(t *testing.T }) assertPlaceResultsHavePreviousAllocs(t, 10, r.place) } + +// Tests rescheduling failed service allocations with desired state stop +func TestReconciler_ForceReschedule_Service(t *testing.T) { + require := require.New(t) + + // Set desired 5 + job := mock.Job() + job.TaskGroups[0].Count = 5 + tgName := job.TaskGroups[0].Name + + // Set up reschedule policy and update stanza + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 24 * time.Hour, + Delay: 5 * time.Second, + DelayFunction: "", + MaxDelay: 1 * time.Hour, + Unlimited: false, + } + job.TaskGroups[0].Update = noCanaryUpdate + + // Create 5 existing allocations + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + allocs = append(allocs, alloc) + alloc.ClientStatus = structs.AllocClientStatusRunning + } + + // Mark one as failed and past its reschedule limit so not eligible to reschedule + allocs[0].ClientStatus = structs.AllocClientStatusFailed + allocs[0].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{ + {RescheduleTime: time.Now().Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + }, + }} + + // Mark DesiredTransition ForceReschedule + allocs[0].DesiredTransition = structs.DesiredTransition{ForceReschedule: helper.BoolToPtr(true)} + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil, "") + r := reconciler.Compute() + + // Verify that no follow up evals were created + evals := r.desiredFollowupEvals[tgName] + require.Nil(evals) + + // Verify that one rescheduled alloc was created because of the forced reschedule + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: nil, + place: 1, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Place: 1, + Ignore: 4, + }, + }, + }) + + // Rescheduled allocs should have previous allocs + assertNamesHaveIndexes(t, intRange(0, 0), placeResultsToNames(r.place)) + assertPlaceResultsHavePreviousAllocs(t, 1, r.place) + assertPlacementsAreRescheduled(t, 1, r.place) +} From 1bad7196129e61bce5cb343e947f63e9763e034d Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 9 May 2018 15:04:27 -0500 Subject: [PATCH 04/14] Added CLI for evaluating job given ID, and modified client API for evaluate to take a request payload --- api/jobs.go | 21 +++++- api/jobs_test.go | 4 +- command/agent/job_endpoint.go | 25 +++++-- command/agent/job_endpoint_test.go | 51 ++++++++++++++ command/commands.go | 5 ++ command/job_eval.go | 106 +++++++++++++++++++++++++++++ command/job_eval_test.go | 65 ++++++++++++++++++ nomad/job_endpoint_test.go | 1 + nomad/structs/structs.go | 1 + 9 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 command/job_eval.go create mode 100644 command/job_eval_test.go diff --git a/api/jobs.go b/api/jobs.go index d3a255d06b8..9b9108f1903 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -224,9 +224,14 @@ func (j *Jobs) Deregister(jobID string, purge bool, q *WriteOptions) (string, *W } // ForceEvaluate is used to force-evaluate an existing job. -func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, error) { +func (j *Jobs) ForceEvaluate(jobID string, opts EvalOptions, q *WriteOptions) (string, *WriteMeta, error) { + req := &JobEvaluateRequest{ + JobID: jobID, + EvalOptions: opts, + } + var resp JobRegisterResponse - wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", nil, &resp, q) + wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", req, &resp, q) if err != nil { return "", nil, err } @@ -1032,3 +1037,15 @@ type JobStabilityResponse struct { JobModifyIndex uint64 WriteMeta } + +// JobEvaluateRequest is used when we just need to re-evaluate a target job +type JobEvaluateRequest struct { + JobID string + EvalOptions EvalOptions + WriteRequest +} + +// EvalOptions is used to encapsulate options when forcing a job evaluation +type EvalOptions struct { + ForceReschedule bool +} diff --git a/api/jobs_test.go b/api/jobs_test.go index dfefde3b41e..9cf3227eb0e 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -1049,7 +1049,7 @@ func TestJobs_ForceEvaluate(t *testing.T) { jobs := c.Jobs() // Force-eval on a non-existent job fails - _, _, err := jobs.ForceEvaluate("job1", nil) + _, _, err := jobs.ForceEvaluate("job1", EvalOptions{}, nil) if err == nil || !strings.Contains(err.Error(), "not found") { t.Fatalf("expected not found error, got: %#v", err) } @@ -1062,7 +1062,7 @@ func TestJobs_ForceEvaluate(t *testing.T) { assertWriteMeta(t, wm) // Try force-eval again - evalID, wm, err := jobs.ForceEvaluate("job1", nil) + evalID, wm, err := jobs.ForceEvaluate("job1", EvalOptions{}, nil) if err != nil { t.Fatalf("err: %s", err) } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index f69f763c82d..03bc8f96de9 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -90,14 +90,25 @@ func (s *HTTPServer) jobForceEvaluate(resp http.ResponseWriter, req *http.Reques if req.Method != "PUT" && req.Method != "POST" { return nil, CodedError(405, ErrInvalidMethod) } + var args structs.JobEvaluateRequest - evalOptions := structs.EvalOptions{} - if _, ok := req.URL.Query()["force"]; ok { - evalOptions.ForceReschedule = true - } - args := structs.JobEvaluateRequest{ - JobID: jobName, - EvalOptions: evalOptions, + // TODO(preetha) remove in 0.9 + // For backwards compatibility allow using this endpoint without a payload + if req.ContentLength == 0 { + args = structs.JobEvaluateRequest{ + JobID: jobName, + } + } else { + if err := decodeBody(req, &args); err != nil { + return nil, CodedError(400, err.Error()) + } + if args.JobID == "" { + return nil, CodedError(400, "Job ID must be specified") + } + + if jobName != "" && args.JobID != jobName { + return nil, CodedError(400, "JobID not same as job name") + } } s.parseWriteRequest(req, &args.WriteRequest) diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index d92433f4ed7..5908c70b81c 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -609,6 +609,57 @@ func TestHTTP_JobForceEvaluate(t *testing.T) { }) } +func TestHTTP_JobEvaluate_ForceReschedule(t *testing.T) { + t.Parallel() + httpTest(t, nil, func(s *TestAgent) { + // Create the job + job := mock.Job() + args := structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var resp structs.JobRegisterResponse + if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { + t.Fatalf("err: %v", err) + } + jobEvalReq := api.JobEvaluateRequest{ + JobID: job.ID, + EvalOptions: api.EvalOptions{ + ForceReschedule: true, + }, + } + + buf := encodeReq(jobEvalReq) + + // Make the HTTP request + req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/evaluate", buf) + if err != nil { + t.Fatalf("err: %v", err) + } + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.JobSpecificRequest(respW, req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Check the response + reg := obj.(structs.JobRegisterResponse) + if reg.EvalID == "" { + t.Fatalf("bad: %v", reg) + } + + // Check for the index + if respW.HeaderMap.Get("X-Nomad-Index") == "" { + t.Fatalf("missing index") + } + }) +} + func TestHTTP_JobEvaluations(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { diff --git a/command/commands.go b/command/commands.go index c4104ca4548..27f33b15913 100644 --- a/command/commands.go +++ b/command/commands.go @@ -270,6 +270,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "job eval": func() (cli.Command, error) { + return &JobEvalCommand{ + Meta: meta, + }, nil + }, "job history": func() (cli.Command, error) { return &JobHistoryCommand{ Meta: meta, diff --git a/command/job_eval.go b/command/job_eval.go new file mode 100644 index 00000000000..fb185176f97 --- /dev/null +++ b/command/job_eval.go @@ -0,0 +1,106 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/api/contexts" + "github.com/posener/complete" +) + +type JobEvalCommand struct { + Meta + forceRescheduling bool +} + +func (c *JobEvalCommand) Help() string { + helpText := ` +Usage: nomad job eval [options] + + Force an evaluation of the provided job id + +General Options: + + ` + generalOptionsUsage() + ` + +Eval Options: + + -force-reschedule + Force reschedule any failed allocations even if they are not currently + eligible for rescheduling +` + return strings.TrimSpace(helpText) +} + +func (c *JobEvalCommand) Synopsis() string { + return "Force evaluating a job using its job id" +} + +func (c *JobEvalCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-force-reschedule": complete.PredictNothing, + }) +} + +func (c *JobEvalCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictFunc(func(a complete.Args) []string { + client, err := c.Meta.Client() + if err != nil { + return nil + } + + resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Jobs, nil) + if err != nil { + return []string{} + } + return resp.Matches[contexts.Jobs] + }) +} + +func (c *JobEvalCommand) Name() string { return "eval" } + +func (c *JobEvalCommand) Run(args []string) int { + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&c.forceRescheduling, "force-reschedule", false, "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we either got no jobs or exactly one. + args = flags.Args() + if len(args) > 1 { + c.Ui.Error("This command takes either no arguments or one: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + if len(args) == 0 { + c.Ui.Error("Must provide a job ID") + return 1 + } + + // Call eval end point + jobID := args[0] + + opts := api.EvalOptions{ + ForceReschedule: c.forceRescheduling, + } + evalId, _, err := client.Jobs().ForceEvaluate(jobID, opts, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error evaluating job: %s", err)) + return 1 + } + c.Ui.Output(fmt.Sprintf("Created eval ID: %q ", evalId)) + return 0 +} diff --git a/command/job_eval_test.go b/command/job_eval_test.go new file mode 100644 index 00000000000..53c22b9e65b --- /dev/null +++ b/command/job_eval_test.go @@ -0,0 +1,65 @@ +package command + +import ( + "strings" + "testing" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/mitchellh/cli" + "github.com/posener/complete" + "github.com/stretchr/testify/assert" +) + +func TestJobEvalCommand_Implements(t *testing.T) { + t.Parallel() + var _ cli.Command = &JobEvalCommand{} +} + +func TestJobEvalCommand_Fails(t *testing.T) { + t.Parallel() + ui := new(cli.MockUi) + cmd := &JobEvalCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { + t.Fatalf("expected exit code 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { + t.Fatalf("expected help output, got: %s", out) + } + ui.ErrorWriter.Reset() + + // Fails when job ID is not specified + if code := cmd.Run([]string{}); code != 1 { + t.Fatalf("expect exit 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, "Must provide a job ID") { + t.Fatalf("unexpected error: %v", out) + } + ui.ErrorWriter.Reset() + +} + +func TestJobEvalCommand_AutocompleteArgs(t *testing.T) { + assert := assert.New(t) + t.Parallel() + + srv, _, url := testServer(t, true, nil) + defer srv.Shutdown() + + ui := new(cli.MockUi) + cmd := &JobEvalCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Create a fake job + state := srv.Agent.Server().State() + j := mock.Job() + assert.Nil(state.UpsertJob(1000, j)) + + prefix := j.ID[:len(j.ID)-5] + args := complete.Args{Last: prefix} + predictor := cmd.AutocompleteArgs() + + res := predictor.Predict(args) + assert.Equal(1, len(res)) + assert.Equal(j.ID, res[0]) +} diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 85197ab8043..fe51fbabee2 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -1325,6 +1325,7 @@ func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { state := s1.fsm.State() job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID) + require.Nil(err) // Create a failed alloc alloc := mock.Alloc() diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index c1d6df810fb..3c73da084ff 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -470,6 +470,7 @@ type JobEvaluateRequest struct { WriteRequest } +// EvalOptions is used to encapsulate options when forcing a job evaluation type EvalOptions struct { ForceReschedule bool } From 4f9d92cad3222411e475592cfe8d70fe9ea368f6 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 9 May 2018 16:01:34 -0500 Subject: [PATCH 05/14] fix test comment --- scheduler/reconcile_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler/reconcile_test.go b/scheduler/reconcile_test.go index 8ac0db02020..3aded05b93c 100644 --- a/scheduler/reconcile_test.go +++ b/scheduler/reconcile_test.go @@ -4571,7 +4571,7 @@ func TestReconciler_SuccessfulDeploymentWithFailedAllocs_Reschedule(t *testing.T assertPlaceResultsHavePreviousAllocs(t, 10, r.place) } -// Tests rescheduling failed service allocations with desired state stop +// Tests force rescheduling a failed alloc that is past its reschedule limit func TestReconciler_ForceReschedule_Service(t *testing.T) { require := require.New(t) From 2d0e273c4354ca92ebe86018b581a6b56862f100 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 9 May 2018 16:51:58 -0500 Subject: [PATCH 06/14] Documentation for evaluate endpoint --- website/source/api/jobs.html.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index 284830937d8..55005c96e14 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -1361,7 +1361,9 @@ $ curl \ ## Create Job Evaluation This endpoint creates a new evaluation for the given job. This can be used to -force run the scheduling logic if necessary. +force run the scheduling logic if necessary. Since Nomad 0.8.4, this endpoint +supports a JSON payload with additional options. Support for calling this end point +without a JSON payload will be removed in Nomad 0.9. | Method | Path | Produces | | ------- | -------------------------- | -------------------------- | @@ -1380,11 +1382,30 @@ The table below shows this endpoint's support for - `:job_id` `(string: )` - Specifies the ID of the job (as specified in the job file during submission). This is specified as part of the path. +- `JobID` `(string: )` - Specify the ID of the job in the JSON payload + +- `EvalOptions` `()` - Specify additional options to be used during the forced evaluation. + - `ForceReschedule` `(bool: false)` - If set, any failed allocations of the job are rescheduled + immediately. This is useful for operators to force immediate placement even if the failed allocations are past + their reschedule limit, or are delayed by several hours because the allocation's reschedule policy has exponential delay. + +### Sample Payload + +```json +{ + "JobID": "my-job", + "EvalOptions": { + "ForceReschedule":true + } +} +``` + ### Sample Request ```text $ curl \ --request POST \ + -d@sample.json \ https://localhost:4646/v1/job/my-job/evaluate ``` From b5e18b6cc2a390927f3a1db1b47eeb8bf43b0098 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 9 May 2018 17:19:23 -0500 Subject: [PATCH 07/14] Docs for job eval CLI --- website/source/docs/commands/job.html.md.erb | 2 + .../source/docs/commands/job/eval.html.md.erb | 48 +++++++++++++++++++ website/source/layouts/docs.erb | 3 ++ 3 files changed, 53 insertions(+) create mode 100644 website/source/docs/commands/job/eval.html.md.erb diff --git a/website/source/docs/commands/job.html.md.erb b/website/source/docs/commands/job.html.md.erb index 1677fd51486..acaccf7a12a 100644 --- a/website/source/docs/commands/job.html.md.erb +++ b/website/source/docs/commands/job.html.md.erb @@ -19,6 +19,7 @@ subcommands are available: * [`job deployments`][deployments] - List deployments for a job * [`job dispatch`][dispatch] - Dispatch an instance of a parameterized job +* [`job eval`][eval] - Force an evaluation for a job * [`job history`][history] - Display all tracked versions of a job * [`job promote`][promote] - Promote a job's canaries * [`job revert`][revert] - Revert to a prior version of the job @@ -26,6 +27,7 @@ subcommands are available: [deployments]: /docs/commands/job/deployments.html "List deployments for a job" [dispatch]: /docs/commands/job/dispatch.html "Dispatch an instance of a parameterized job" +[eval]: /docs/commands/job/eval.html "Force an evaluation for a job" [history]: /docs/commands/job/history.html "Display all tracked versions of a job" [promote]: /docs/commands/job/promote.html "Promote a job's canaries" [revert]: /docs/commands/job/revert.html "Revert to a prior version of the job" diff --git a/website/source/docs/commands/job/eval.html.md.erb b/website/source/docs/commands/job/eval.html.md.erb new file mode 100644 index 00000000000..4300d527c90 --- /dev/null +++ b/website/source/docs/commands/job/eval.html.md.erb @@ -0,0 +1,48 @@ +--- +layout: "docs" +page_title: "Commands: job eval" +sidebar_current: "docs-commands-job-eval" +description: > + The job eval command is used to force an evaluation of a job +--- + +# Command: job eval + +The `job eval` command is used to force an evaluation of a job, given the job ID. + +## Usage + +``` +nomad job eval [options] +``` + +The `job eval` command requires a single argument, specifying the job ID to evaluate. +If there is an exact match based on the provided job ID, then +the job will be evaluated, forcing a scheduler run. + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## Eval Options + +* `-force-reschedule`: `force-reschedule` is used to force placement of any failed allocations. +If this is set, failed allocations that are past their reschedule limit, as well as any that are +scheduled to be replaced at a future time are placed immediately. This option only places failed +allocations if the task group has rescheduling enabled. + +## Examples + +Evaluate the job with ID "job1": + +``` +$ nomad job eval job1 +Created eval ID: "6754c2e3-9abb-e7e9-dc92-76aab01751c8" +``` + +Evaluate the job with ID "job1", and reschedule any eligible failed allocations: + +``` +$ nomad job eval -force-reschedule job1 +Created eval ID: "6754c2e3-9abb-e7e9-dc92-76aab01751c8" +``` \ No newline at end of file diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 4f48b934069..b0e3b7250fc 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -231,6 +231,9 @@ > dispatch + > + eval + > history From 879c2c93c9201f15ee535858c61c209e542a589d Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Thu, 10 May 2018 14:42:24 -0500 Subject: [PATCH 08/14] Code review feedback --- command/agent/job_endpoint.go | 4 +- command/job_eval.go | 23 +++---- command/job_eval_test.go | 64 ++++++++++++++++++- nomad/job_endpoint.go | 4 +- website/source/api/jobs.html.md | 4 +- .../source/docs/commands/job/eval.html.md.erb | 6 +- 6 files changed, 82 insertions(+), 23 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 03bc8f96de9..5976c7c985e 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -92,8 +92,8 @@ func (s *HTTPServer) jobForceEvaluate(resp http.ResponseWriter, req *http.Reques } var args structs.JobEvaluateRequest - // TODO(preetha) remove in 0.9 - // For backwards compatibility allow using this endpoint without a payload + // TODO(preetha): remove in 0.9 + // COMPAT: For backwards compatibility allow using this endpoint without a payload if req.ContentLength == 0 { args = structs.JobEvaluateRequest{ JobID: jobName, diff --git a/command/job_eval.go b/command/job_eval.go index fb185176f97..056b2165abd 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -18,7 +18,9 @@ func (c *JobEvalCommand) Help() string { helpText := ` Usage: nomad job eval [options] - Force an evaluation of the provided job id + Force an evaluation of the provided job ID. Forcing an evaluation will trigger the scheduler + to re-evaluate the job. The force flags allow operators to force the scheduler to create + new allocations under certain scenarios. General Options: @@ -27,14 +29,14 @@ General Options: Eval Options: -force-reschedule - Force reschedule any failed allocations even if they are not currently - eligible for rescheduling + Force reschedule failed allocations even if they are not currently + eligible for rescheduling. ` return strings.TrimSpace(helpText) } func (c *JobEvalCommand) Synopsis() string { - return "Force evaluating a job using its job id" + return "Force an evaluation for the job using its job ID" } func (c *JobEvalCommand) AutocompleteFlags() complete.Flags { @@ -59,7 +61,7 @@ func (c *JobEvalCommand) AutocompleteArgs() complete.Predictor { }) } -func (c *JobEvalCommand) Name() string { return "eval" } +func (c *JobEvalCommand) Name() string { return "job eval" } func (c *JobEvalCommand) Run(args []string) int { flags := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -72,8 +74,8 @@ func (c *JobEvalCommand) Run(args []string) int { // Check that we either got no jobs or exactly one. args = flags.Args() - if len(args) > 1 { - c.Ui.Error("This command takes either no arguments or one: ") + if len(args) != 1 { + c.Ui.Error("This command takes one argument: ") c.Ui.Error(commandErrorText(c)) return 1 } @@ -85,12 +87,7 @@ func (c *JobEvalCommand) Run(args []string) int { return 1 } - if len(args) == 0 { - c.Ui.Error("Must provide a job ID") - return 1 - } - - // Call eval end point + // Call eval endpoint jobID := args[0] opts := api.EvalOptions{ diff --git a/command/job_eval_test.go b/command/job_eval_test.go index 53c22b9e65b..f6f8b278692 100644 --- a/command/job_eval_test.go +++ b/command/job_eval_test.go @@ -4,10 +4,15 @@ import ( "strings" "testing" + "fmt" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" "github.com/posener/complete" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestJobEvalCommand_Implements(t *testing.T) { @@ -33,13 +38,70 @@ func TestJobEvalCommand_Fails(t *testing.T) { if code := cmd.Run([]string{}); code != 1 { t.Fatalf("expect exit 1, got: %d", code) } - if out := ui.ErrorWriter.String(); !strings.Contains(out, "Must provide a job ID") { + if out := ui.ErrorWriter.String(); !strings.Contains(out, "This command takes one argument") { t.Fatalf("unexpected error: %v", out) } ui.ErrorWriter.Reset() } +func TestJobEvalCommand_Run(t *testing.T) { + t.Parallel() + srv, client, url := testServer(t, true, nil) + defer srv.Shutdown() + + // Wait for a node to be ready + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + for _, node := range nodes { + if node.Status == structs.NodeStatusReady { + return true, nil + } + } + return false, fmt.Errorf("no ready nodes") + }, func(err error) { + t.Fatalf("err: %v", err) + }) + + ui := new(cli.MockUi) + cmd := &JobEvalCommand{Meta: Meta{Ui: ui}} + require := require.New(t) + + state := srv.Agent.Server().State() + + // Create a job + job := mock.Job() + err := state.UpsertJob(11, job) + require.Nil(err) + + job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID) + require.Nil(err) + + // Create a failed alloc for the job + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.TaskGroup = job.TaskGroups[0].Name + alloc.Namespace = job.Namespace + alloc.ClientStatus = structs.AllocClientStatusFailed + err = state.UpsertAllocs(12, []*structs.Allocation{alloc}) + require.Nil(err) + + if code := cmd.Run([]string{"-address=" + url, "-force-reschedule", job.ID}); code != 0 { + t.Fatalf("expected exit 0, got: %d", code) + } + + // Lookup alloc again + alloc, err = state.AllocByID(nil, alloc.ID) + require.NotNil(alloc) + require.Nil(err) + require.True(*alloc.DesiredTransition.ForceReschedule) + +} + func TestJobEvalCommand_AutocompleteArgs(t *testing.T) { assert := assert.New(t) t.Parallel() diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index c6c12106932..164899e5873 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -557,11 +557,11 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis for _, alloc := range allocs { taskGroup := job.LookupTaskGroup(alloc.TaskGroup) // Forcing rescheduling is only allowed if task group has rescheduling enabled - if taskGroup == nil || taskGroup.ReschedulePolicy == nil || !taskGroup.ReschedulePolicy.Enabled() { + if taskGroup == nil || !taskGroup.ReschedulePolicy.Enabled() { continue } - if alloc.NextAllocation == "" && alloc.ClientStatus == structs.AllocClientStatusFailed { + if alloc.NextAllocation == "" && alloc.ClientStatus == structs.AllocClientStatusFailed && !alloc.DesiredTransition.ShouldForceReschedule() { forceRescheduleAllocs[alloc.ID] = allowForceRescheduleTransition } } diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index 55005c96e14..a1270fe70ea 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -1385,7 +1385,7 @@ The table below shows this endpoint's support for - `JobID` `(string: )` - Specify the ID of the job in the JSON payload - `EvalOptions` `()` - Specify additional options to be used during the forced evaluation. - - `ForceReschedule` `(bool: false)` - If set, any failed allocations of the job are rescheduled + - `ForceReschedule` `(bool: false)` - If set, failed allocations of the job are rescheduled immediately. This is useful for operators to force immediate placement even if the failed allocations are past their reschedule limit, or are delayed by several hours because the allocation's reschedule policy has exponential delay. @@ -1405,7 +1405,7 @@ The table below shows this endpoint's support for ```text $ curl \ --request POST \ - -d@sample.json \ + -d @sample.json \ https://localhost:4646/v1/job/my-job/evaluate ``` diff --git a/website/source/docs/commands/job/eval.html.md.erb b/website/source/docs/commands/job/eval.html.md.erb index 4300d527c90..b8e84f05c96 100644 --- a/website/source/docs/commands/job/eval.html.md.erb +++ b/website/source/docs/commands/job/eval.html.md.erb @@ -13,7 +13,7 @@ The `job eval` command is used to force an evaluation of a job, given the job ID ## Usage ``` -nomad job eval [options] +nomad job eval [options] ``` The `job eval` command requires a single argument, specifying the job ID to evaluate. @@ -26,8 +26,8 @@ the job will be evaluated, forcing a scheduler run. ## Eval Options -* `-force-reschedule`: `force-reschedule` is used to force placement of any failed allocations. -If this is set, failed allocations that are past their reschedule limit, as well as any that are +* `-force-reschedule`: `force-reschedule` is used to force placement of failed allocations. +If this is set, failed allocations that are past their reschedule limit, and those that are scheduled to be replaced at a future time are placed immediately. This option only places failed allocations if the task group has rescheduling enabled. From 53c05c590dbae2d4303af05ed81162de68e90154 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Thu, 10 May 2018 15:02:58 -0500 Subject: [PATCH 09/14] Add support for monitoring evals, and -detach/-verbose support --- command/job_eval.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/command/job_eval.go b/command/job_eval.go index 056b2165abd..0cd96bcddea 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -31,6 +31,13 @@ Eval Options: -force-reschedule Force reschedule failed allocations even if they are not currently eligible for rescheduling. + -detach + Return immediately instead of entering monitor mode. After deployment + resume, the evaluation ID will be printed to the screen, which can be used + to examine the evaluation using the eval-status command. + + -verbose + Display full information. ` return strings.TrimSpace(helpText) } @@ -43,6 +50,8 @@ func (c *JobEvalCommand) AutocompleteFlags() complete.Flags { return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-force-reschedule": complete.PredictNothing, + "-detach": complete.PredictNothing, + "-verbose": complete.PredictNothing, }) } @@ -64,9 +73,13 @@ func (c *JobEvalCommand) AutocompleteArgs() complete.Predictor { func (c *JobEvalCommand) Name() string { return "job eval" } func (c *JobEvalCommand) Run(args []string) int { + var detach, verbose bool + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.BoolVar(&c.forceRescheduling, "force-reschedule", false, "") + flags.BoolVar(&detach, "detach", false, "") + flags.BoolVar(&verbose, "verbose", false, "") if err := flags.Parse(args); err != nil { return 1 @@ -87,6 +100,11 @@ func (c *JobEvalCommand) Run(args []string) int { return 1 } + // Truncate the id unless full length is requested + length := shortId + if verbose { + length = fullId + } // Call eval endpoint jobID := args[0] @@ -98,6 +116,12 @@ func (c *JobEvalCommand) Run(args []string) int { c.Ui.Error(fmt.Sprintf("Error evaluating job: %s", err)) return 1 } - c.Ui.Output(fmt.Sprintf("Created eval ID: %q ", evalId)) + c.Ui.Output(fmt.Sprintf("Created eval ID: %q ", limit(evalId, length))) + if detach { + return 0 + } + + mon := newMonitor(c.Ui, client, length) + return mon.monitor(evalId, false) return 0 } From e2f13d28073a4e6ac1b744c6930a152d26bb63d0 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Thu, 10 May 2018 15:30:44 -0500 Subject: [PATCH 10/14] unit test for job eval should detach --- command/job_eval_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/job_eval_test.go b/command/job_eval_test.go index f6f8b278692..ef5019e360e 100644 --- a/command/job_eval_test.go +++ b/command/job_eval_test.go @@ -90,7 +90,7 @@ func TestJobEvalCommand_Run(t *testing.T) { err = state.UpsertAllocs(12, []*structs.Allocation{alloc}) require.Nil(err) - if code := cmd.Run([]string{"-address=" + url, "-force-reschedule", job.ID}); code != 0 { + if code := cmd.Run([]string{"-address=" + url, "-force-reschedule", "-detach", job.ID}); code != 0 { t.Fatalf("expected exit 0, got: %d", code) } From b2006cc784f035298d8a8477039452829f29c4be Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Fri, 11 May 2018 13:39:55 -0500 Subject: [PATCH 11/14] more review feedback --- command/job_eval.go | 12 +++++++----- website/source/docs/commands/job/eval.html.md.erb | 6 ++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/command/job_eval.go b/command/job_eval.go index 0cd96bcddea..82b2e83121c 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -31,10 +31,11 @@ Eval Options: -force-reschedule Force reschedule failed allocations even if they are not currently eligible for rescheduling. + -detach - Return immediately instead of entering monitor mode. After deployment - resume, the evaluation ID will be printed to the screen, which can be used - to examine the evaluation using the eval-status command. + Return immediately instead of entering monitor mode. The ID + of the evaluation created will be printed to the screen, which can be + used to examine the evaluation using the eval-status command. -verbose Display full information. @@ -43,7 +44,7 @@ Eval Options: } func (c *JobEvalCommand) Synopsis() string { - return "Force an evaluation for the job using its job ID" + return "Force an evaluation for the job" } func (c *JobEvalCommand) AutocompleteFlags() complete.Flags { @@ -116,8 +117,9 @@ func (c *JobEvalCommand) Run(args []string) int { c.Ui.Error(fmt.Sprintf("Error evaluating job: %s", err)) return 1 } - c.Ui.Output(fmt.Sprintf("Created eval ID: %q ", limit(evalId, length))) + if detach { + c.Ui.Output(fmt.Sprintf("Created eval ID: %q ", limit(evalId, length))) return 0 } diff --git a/website/source/docs/commands/job/eval.html.md.erb b/website/source/docs/commands/job/eval.html.md.erb index b8e84f05c96..c3fcbe7216a 100644 --- a/website/source/docs/commands/job/eval.html.md.erb +++ b/website/source/docs/commands/job/eval.html.md.erb @@ -31,6 +31,12 @@ If this is set, failed allocations that are past their reschedule limit, and tho scheduled to be replaced at a future time are placed immediately. This option only places failed allocations if the task group has rescheduling enabled. +* `-detach`: Return immediately instead of monitoring. A new evaluation ID + will be output, which can be used to examine the evaluation using the + [eval status](/docs/commands/eval-status.html) command + +* `-verbose`: Show full information. + ## Examples Evaluate the job with ID "job1": From ae5d8fd593c85c11083d793953b48d6859249831 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Fri, 11 May 2018 14:18:53 -0500 Subject: [PATCH 12/14] Add new method EvaluateWithOptions to avoid breaking go API client --- api/jobs.go | 12 +++++++++++- api/jobs_test.go | 4 ++-- command/job_eval.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 9b9108f1903..9278f41a3ed 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -224,7 +224,17 @@ func (j *Jobs) Deregister(jobID string, purge bool, q *WriteOptions) (string, *W } // ForceEvaluate is used to force-evaluate an existing job. -func (j *Jobs) ForceEvaluate(jobID string, opts EvalOptions, q *WriteOptions) (string, *WriteMeta, error) { +func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, error) { + var resp JobRegisterResponse + wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", nil, &resp, q) + if err != nil { + return "", nil, err + } + return resp.EvalID, wm, nil +} + +// ForceEvaluate is used to force-evaluate an existing job. +func (j *Jobs) EvaluateWithOpts(jobID string, opts EvalOptions, q *WriteOptions) (string, *WriteMeta, error) { req := &JobEvaluateRequest{ JobID: jobID, EvalOptions: opts, diff --git a/api/jobs_test.go b/api/jobs_test.go index 9cf3227eb0e..dfefde3b41e 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -1049,7 +1049,7 @@ func TestJobs_ForceEvaluate(t *testing.T) { jobs := c.Jobs() // Force-eval on a non-existent job fails - _, _, err := jobs.ForceEvaluate("job1", EvalOptions{}, nil) + _, _, err := jobs.ForceEvaluate("job1", nil) if err == nil || !strings.Contains(err.Error(), "not found") { t.Fatalf("expected not found error, got: %#v", err) } @@ -1062,7 +1062,7 @@ func TestJobs_ForceEvaluate(t *testing.T) { assertWriteMeta(t, wm) // Try force-eval again - evalID, wm, err := jobs.ForceEvaluate("job1", EvalOptions{}, nil) + evalID, wm, err := jobs.ForceEvaluate("job1", nil) if err != nil { t.Fatalf("err: %s", err) } diff --git a/command/job_eval.go b/command/job_eval.go index 82b2e83121c..351ba10c309 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -112,7 +112,7 @@ func (c *JobEvalCommand) Run(args []string) int { opts := api.EvalOptions{ ForceReschedule: c.forceRescheduling, } - evalId, _, err := client.Jobs().ForceEvaluate(jobID, opts, nil) + evalId, _, err := client.Jobs().EvaluateWithOpts(jobID, opts, nil) if err != nil { c.Ui.Error(fmt.Sprintf("Error evaluating job: %s", err)) return 1 From 2ea09b82a00a81f9f5aea2421662543d30c136f6 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Mon, 21 May 2018 17:20:59 -0500 Subject: [PATCH 13/14] Fix docs and method documentation in API --- api/jobs.go | 3 ++- .../source/docs/commands/job/eval.html.md.erb | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 9278f41a3ed..2543229770d 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -233,7 +233,8 @@ func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, return resp.EvalID, wm, nil } -// ForceEvaluate is used to force-evaluate an existing job. +// EvaluateWithOpts is used to force-evaluate an existing job and takes additional options +// for whether to force reschedule failed allocations func (j *Jobs) EvaluateWithOpts(jobID string, opts EvalOptions, q *WriteOptions) (string, *WriteMeta, error) { req := &JobEvaluateRequest{ JobID: jobID, diff --git a/website/source/docs/commands/job/eval.html.md.erb b/website/source/docs/commands/job/eval.html.md.erb index c3fcbe7216a..8ede561c166 100644 --- a/website/source/docs/commands/job/eval.html.md.erb +++ b/website/source/docs/commands/job/eval.html.md.erb @@ -43,12 +43,27 @@ Evaluate the job with ID "job1": ``` $ nomad job eval job1 -Created eval ID: "6754c2e3-9abb-e7e9-dc92-76aab01751c8" +==> Monitoring evaluation "0f3bc0f3" + Evaluation triggered by job "test" + Evaluation within deployment: "51baf5c8" + Evaluation status changed: "pending" -> "complete" +==> Evaluation "0f3bc0f3" finished with status "complete" +``` + +Evaluate the job with ID "job1" and return immediately: + +``` +$ nomad job eval -detach job1 +Created eval ID: "4947e728" ``` Evaluate the job with ID "job1", and reschedule any eligible failed allocations: ``` $ nomad job eval -force-reschedule job1 -Created eval ID: "6754c2e3-9abb-e7e9-dc92-76aab01751c8" +==> Monitoring evaluation "0f3bc0f3" + Evaluation triggered by job "test" + Evaluation within deployment: "51baf5c8" + Evaluation status changed: "pending" -> "complete" +==> Evaluation "0f3bc0f3" finished with status "complete" ``` \ No newline at end of file From a5ca3790a51a13e417f3b5eae9e951429d6e27e2 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Mon, 21 May 2018 18:00:14 -0500 Subject: [PATCH 14/14] remove extra return --- command/job_eval.go | 1 - 1 file changed, 1 deletion(-) diff --git a/command/job_eval.go b/command/job_eval.go index 351ba10c309..2b3f4e64805 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -125,5 +125,4 @@ func (c *JobEvalCommand) Run(args []string) int { mon := newMonitor(c.Ui, client, length) return mon.monitor(evalId, false) - return 0 }