Allow configurable range of Job priorities

.changelog/16084.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: Allow configurable range of Job priorities
+```

api/jobs.go

@@ -938,7 +938,7 @@ func (j *Job) Canonicalize() {
 		j.Namespace = pointerOf(DefaultNamespace)
 	}
 	if j.Priority == nil {
-		j.Priority = pointerOf(50)
+		j.Priority = pointerOf(0)
 	}
 	if j.Stop == nil {
 		j.Stop = pointerOf(false)

api/jobs_test.go

@@ -279,7 +279,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Namespace:         pointerOf(DefaultNamespace),
 				Type:              pointerOf("service"),
 				ParentID:          pointerOf(""),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -374,7 +374,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Namespace:         pointerOf(DefaultNamespace),
 				Type:              pointerOf("batch"),
 				ParentID:          pointerOf(""),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -452,7 +452,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
 				ParentID:          pointerOf("lol"),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -621,7 +621,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				ID:                pointerOf("example_template"),
 				Name:              pointerOf("example_template"),
 				ParentID:          pointerOf(""),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
 				AllAtOnce:         pointerOf(false),
@@ -791,7 +791,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Name:              pointerOf("bar"),
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -882,7 +882,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
 				ParentID:          pointerOf("lol"),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -1058,7 +1058,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
 				ParentID:          pointerOf("lol"),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),
@@ -1229,7 +1229,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				Region:            pointerOf("global"),
 				Type:              pointerOf("service"),
 				ParentID:          pointerOf("lol"),
-				Priority:          pointerOf(50),
+				Priority:          pointerOf(0),
 				AllAtOnce:         pointerOf(false),
 				ConsulToken:       pointerOf(""),
 				ConsulNamespace:   pointerOf(""),

command/agent/agent.go

@@ -326,6 +326,23 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		}
 	}
 
+	jobMaxPriority := structs.JobDefaultMaxPriority
+	if agentConfig.Server.JobMaxPriority != nil && *agentConfig.Server.JobMaxPriority != 0 {
+		jobMaxPriority = *agentConfig.Server.JobMaxPriority
+		if jobMaxPriority < structs.JobDefaultMaxPriority || jobMaxPriority > structs.JobMaxPriority {
+			return nil, fmt.Errorf("job_max_priority cannot be %d. Must be between %d and %d", *agentConfig.Server.JobMaxPriority, structs.JobDefaultMaxPriority, structs.JobMaxPriority)
+		}
+	}
+	jobDefaultPriority := structs.JobDefaultPriority
+	if agentConfig.Server.JobDefaultPriority != nil && *agentConfig.Server.JobDefaultPriority != 0 {
+		jobDefaultPriority = *agentConfig.Server.JobDefaultPriority
+		if jobDefaultPriority < structs.JobDefaultPriority || jobDefaultPriority >= jobMaxPriority {
+			return nil, fmt.Errorf("job_default_priority cannot be %d. Must be between %d and %d", *agentConfig.Server.JobDefaultPriority, structs.JobDefaultPriority, jobMaxPriority)
+		}
+	}
+	conf.JobMaxPriority = jobMaxPriority
+	conf.JobDefaultPriority = jobDefaultPriority
+
 	// Set up the bind addresses
 	rpcAddr, err := net.ResolveTCPAddr("tcp", agentConfig.normalizedAddrs.RPC)
 	if err != nil {

command/agent/agent_test.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
@@ -1569,3 +1570,144 @@ func TestAgent_ProxyRPC_Dev(t *testing.T) {
 		})
 
 }
+
+func TestAgent_ServerConfig_JobMaxPriority_Ok(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []struct {
+		maxPriority    *int
+		jobMaxPriority int
+	}{
+		{
+			maxPriority:    nil,
+			jobMaxPriority: 100,
+		},
+
+		{
+			maxPriority:    pointer.Of(0),
+			jobMaxPriority: 100,
+		},
+		{
+			maxPriority:    pointer.Of(100),
+			jobMaxPriority: 100,
+		},
+		{
+			maxPriority:    pointer.Of(200),
+			jobMaxPriority: 200,
+		},
+		{
+			maxPriority:    pointer.Of(32766),
+			jobMaxPriority: 32766,
+		},
+	}
+
+	for _, tc := range cases {
+		v := "default"
+		if tc.maxPriority != nil {
+			v = fmt.Sprintf("%v", *tc.maxPriority)
+		}
+		t.Run(v, func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobMaxPriority = tc.maxPriority
+
+			serverConf, err := convertServerConfig(conf)
+			must.NoError(t, err)
+			must.Eq(t, tc.jobMaxPriority, serverConf.JobMaxPriority)
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobMaxPriority_Bad(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []int{
+		99,
+		math.MaxInt16,
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%v", tc), func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobMaxPriority = &tc
+
+			_, err := convertServerConfig(conf)
+			must.Error(t, err)
+			must.ErrorContains(t, err, "job_max_priority cannot be")
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobDefaultPriority_Ok(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []struct {
+		defaultPriority    *int
+		jobDefaultPriority int
+	}{
+		{
+			defaultPriority:    nil,
+			jobDefaultPriority: 50,
+		},
+
+		{
+			defaultPriority:    pointer.Of(0),
+			jobDefaultPriority: 50,
+		},
+		{
+			defaultPriority:    pointer.Of(50),
+			jobDefaultPriority: 50,
+		},
+		{
+			defaultPriority:    pointer.Of(60),
+			jobDefaultPriority: 60,
+		},
+		{
+			defaultPriority:    pointer.Of(99),
+			jobDefaultPriority: 99,
+		},
+	}
+
+	for _, tc := range cases {
+		v := "default"
+		if tc.defaultPriority != nil {
+			v = fmt.Sprintf("%v", *tc.defaultPriority)
+		}
+		t.Run(v, func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobDefaultPriority = tc.defaultPriority
+
+			serverConf, err := convertServerConfig(conf)
+			must.NoError(t, err)
+
+			must.Eq(t, tc.jobDefaultPriority, serverConf.JobDefaultPriority)
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobDefaultPriority_Bad(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []int{
+		49,
+		100,
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%v", tc), func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobDefaultPriority = &tc
+
+			_, err := convertServerConfig(conf)
+			must.Error(t, err)
+			must.ErrorContains(t, err, "job_default_priority cannot be")
+		})
+	}
+}

command/agent/config.go

@@ -644,6 +644,12 @@ type ServerConfig struct {
 	// forced to send an entire snapshot. The value passed here is the initial
 	// setting used. This can be tuned during operation using a hot reload.
 	RaftTrailingLogs *int `hcl:"raft_trailing_logs"`
+
+	// JobDefaultPriority is the default Job priority if not specified.
+	JobDefaultPriority *int `hcl:"job_default_priority"`
+
+	// JobMaxPriority is an upper bound on the Job priority.
+	JobMaxPriority *int `hcl:"job_max_priority"`
 }
 
 func (s *ServerConfig) Copy() *ServerConfig {
@@ -669,6 +675,8 @@ func (s *ServerConfig) Copy() *ServerConfig {
 	ns.RaftSnapshotInterval = pointer.Copy(s.RaftSnapshotInterval)
 	ns.RaftSnapshotThreshold = pointer.Copy(s.RaftSnapshotThreshold)
 	ns.RaftTrailingLogs = pointer.Copy(s.RaftTrailingLogs)
+	ns.JobDefaultPriority = pointer.Copy(s.JobDefaultPriority)
+	ns.JobMaxPriority = pointer.Copy(s.JobMaxPriority)
 	return &ns
 }
 
@@ -1867,6 +1875,12 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.JobGCThreshold != "" {
 		result.JobGCThreshold = b.JobGCThreshold
 	}
+	if b.JobDefaultPriority != nil {
+		result.JobDefaultPriority = pointer.Of(*b.JobDefaultPriority)
+	}
+	if b.JobMaxPriority != nil {
+		result.JobMaxPriority = pointer.Of(*b.JobMaxPriority)
+	}
 	if b.EvalGCThreshold != "" {
 		result.EvalGCThreshold = b.EvalGCThreshold
 	}

command/agent/config_parse_test.go

@@ -147,7 +147,9 @@ var basicConfig = &Config{
 				ServiceSchedulerEnabled: true,
 			},
 		},
-		LicensePath: "/tmp/nomad.hclic",
+		LicensePath:        "/tmp/nomad.hclic",
+		JobDefaultPriority: pointer.Of(100),
+		JobMaxPriority:     pointer.Of(200),
 	},
 	ACL: &ACLConfig{
 		Enabled:                  true,

command/agent/config_test.go

@@ -358,6 +358,8 @@ func TestConfig_Merge(t *testing.T) {
 				NodeThreshold: 100,
 				NodeWindow:    11 * time.Minute,
 			},
+			JobMaxPriority:     pointer.Of(200),
+			JobDefaultPriority: pointer.Of(100),
 		},
 		ACL: &ACLConfig{
 			Enabled:               true,

command/agent/job_endpoint.go

@@ -196,6 +196,11 @@ func (s *HTTPServer) ValidateJobRequest(resp http.ResponseWriter, req *http.Requ
 
 	job := ApiJobToStructJob(validateRequest.Job)
 
+	// If the job priority is not set, we fallback on the defaults specified in the server config
+	if job.Priority == 0 {
+		job.Priority = s.agent.Server().GetConfig().JobDefaultPriority
+	}
+
 	args := structs.JobValidateRequest{
 		Job: job,
 		WriteRequest: structs.WriteRequest{
@@ -826,6 +831,11 @@ func (s *HTTPServer) apiJobAndRequestToStructs(job *api.Job, req *http.Request,
 	sJob.Region = jobRegion
 	writeReq.Region = requestRegion
 
+	// If the job priority is not set, we fallback on the defaults specified in the server config
+	if sJob.Priority == 0 {
+		sJob.Priority = s.agent.Server().GetConfig().JobDefaultPriority
+	}
+
 	queryNamespace := req.URL.Query().Get("namespace")
 	namespace := namespaceForJob(job.Namespace, queryNamespace, writeReq.Namespace)
 	sJob.Namespace = namespace