Allow configurable range of Job priorities

.changelog/16084.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: Allow configurable range of Job priorities
+```

api/jobs.go

@@ -31,6 +31,9 @@ const (
 	// DefaultNamespace is the default namespace.
 	DefaultNamespace = "default"
 
+	// DefaultPriority is the default priority
+	DefaultPriority = 50
+
 	// For Job configuration, GlobalRegion is a sentinel region value
 	// that users may specify to indicate the job should be run on
 	// the region of the node that the job was submitted to.
@@ -938,7 +941,7 @@ func (j *Job) Canonicalize() {
 		j.Namespace = pointerOf(DefaultNamespace)
 	}
 	if j.Priority == nil {
-		j.Priority = pointerOf(50)
+		j.Priority = pointerOf(DefaultPriority)
 	}
 	if j.Stop == nil {
 		j.Stop = pointerOf(false)

command/agent/agent.go

@@ -326,6 +326,23 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		}
 	}
 
+	jobMaxPriority := structs.JobDefaultMaxPriority
+	if agentConfig.Server.JobMaxPriority != nil && *agentConfig.Server.JobMaxPriority != 0 {
+		jobMaxPriority = *agentConfig.Server.JobMaxPriority
+		if jobMaxPriority < structs.JobDefaultMaxPriority || jobMaxPriority > structs.JobMaxPriority {
+			return nil, fmt.Errorf("job_max_priority cannot be %d. Must be between %d and %d", *agentConfig.Server.JobMaxPriority, structs.JobDefaultMaxPriority, structs.JobMaxPriority)
+		}
+	}
+	jobDefaultPriority := structs.JobDefaultPriority
+	if agentConfig.Server.JobDefaultPriority != nil && *agentConfig.Server.JobDefaultPriority != 0 {
+		jobDefaultPriority = *agentConfig.Server.JobDefaultPriority
+		if jobDefaultPriority < structs.JobDefaultPriority || jobDefaultPriority >= jobMaxPriority {
+			return nil, fmt.Errorf("job_default_priority cannot be %d. Must be between %d and %d", *agentConfig.Server.JobDefaultPriority, structs.JobDefaultPriority, jobMaxPriority)
+		}
+	}
+	conf.JobMaxPriority = jobMaxPriority
+	conf.JobDefaultPriority = jobDefaultPriority
+
 	// Set up the bind addresses
 	rpcAddr, err := net.ResolveTCPAddr("tcp", agentConfig.normalizedAddrs.RPC)
 	if err != nil {

command/agent/agent_test.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
@@ -1569,3 +1570,144 @@ func TestAgent_ProxyRPC_Dev(t *testing.T) {
 		})
 
 }
+
+func TestAgent_ServerConfig_JobMaxPriority_Ok(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []struct {
+		maxPriority    *int
+		jobMaxPriority int
+	}{
+		{
+			maxPriority:    nil,
+			jobMaxPriority: 100,
+		},
+
+		{
+			maxPriority:    pointer.Of(0),
+			jobMaxPriority: 100,
+		},
+		{
+			maxPriority:    pointer.Of(100),
+			jobMaxPriority: 100,
+		},
+		{
+			maxPriority:    pointer.Of(200),
+			jobMaxPriority: 200,
+		},
+		{
+			maxPriority:    pointer.Of(32766),
+			jobMaxPriority: 32766,
+		},
+	}
+
+	for _, tc := range cases {
+		v := "default"
+		if tc.maxPriority != nil {
+			v = fmt.Sprintf("%v", *tc.maxPriority)
+		}
+		t.Run(v, func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobMaxPriority = tc.maxPriority
+
+			serverConf, err := convertServerConfig(conf)
+			must.NoError(t, err)
+			must.Eq(t, tc.jobMaxPriority, serverConf.JobMaxPriority)
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobMaxPriority_Bad(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []int{
+		99,
+		math.MaxInt16,
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%v", tc), func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobMaxPriority = &tc
+
+			_, err := convertServerConfig(conf)
+			must.Error(t, err)
+			must.ErrorContains(t, err, "job_max_priority cannot be")
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobDefaultPriority_Ok(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []struct {
+		defaultPriority    *int
+		jobDefaultPriority int
+	}{
+		{
+			defaultPriority:    nil,
+			jobDefaultPriority: 50,
+		},
+
+		{
+			defaultPriority:    pointer.Of(0),
+			jobDefaultPriority: 50,
+		},
+		{
+			defaultPriority:    pointer.Of(50),
+			jobDefaultPriority: 50,
+		},
+		{
+			defaultPriority:    pointer.Of(60),
+			jobDefaultPriority: 60,
+		},
+		{
+			defaultPriority:    pointer.Of(99),
+			jobDefaultPriority: 99,
+		},
+	}
+
+	for _, tc := range cases {
+		v := "default"
+		if tc.defaultPriority != nil {
+			v = fmt.Sprintf("%v", *tc.defaultPriority)
+		}
+		t.Run(v, func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobDefaultPriority = tc.defaultPriority
+
+			serverConf, err := convertServerConfig(conf)
+			must.NoError(t, err)
+
+			must.Eq(t, tc.jobDefaultPriority, serverConf.JobDefaultPriority)
+		})
+	}
+}
+
+func TestAgent_ServerConfig_JobDefaultPriority_Bad(t *testing.T) {
+	ci.Parallel(t)
+
+	cases := []int{
+		49,
+		100,
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%v", tc), func(t *testing.T) {
+			conf := DevConfig(nil)
+			must.NoError(t, conf.normalizeAddrs())
+
+			conf.Server.JobDefaultPriority = &tc
+
+			_, err := convertServerConfig(conf)
+			must.Error(t, err)
+			must.ErrorContains(t, err, "job_default_priority cannot be")
+		})
+	}
+}

command/agent/config.go

@@ -644,6 +644,12 @@ type ServerConfig struct {
 	// forced to send an entire snapshot. The value passed here is the initial
 	// setting used. This can be tuned during operation using a hot reload.
 	RaftTrailingLogs *int `hcl:"raft_trailing_logs"`
+
+	// JobDefaultPriority is the default Job priority if not specified.
+	JobDefaultPriority *int `hcl:"job_default_priority"`
+
+	// JobMaxPriority is an upper bound on the Job priority.
+	JobMaxPriority *int `hcl:"job_max_priority"`
 }
 
 func (s *ServerConfig) Copy() *ServerConfig {
@@ -669,6 +675,8 @@ func (s *ServerConfig) Copy() *ServerConfig {
 	ns.RaftSnapshotInterval = pointer.Copy(s.RaftSnapshotInterval)
 	ns.RaftSnapshotThreshold = pointer.Copy(s.RaftSnapshotThreshold)
 	ns.RaftTrailingLogs = pointer.Copy(s.RaftTrailingLogs)
+	ns.JobDefaultPriority = pointer.Copy(s.JobDefaultPriority)
+	ns.JobMaxPriority = pointer.Copy(s.JobMaxPriority)
 	return &ns
 }
 
@@ -1867,6 +1875,12 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.JobGCThreshold != "" {
 		result.JobGCThreshold = b.JobGCThreshold
 	}
+	if b.JobDefaultPriority != nil {
+		result.JobDefaultPriority = pointer.Of(*b.JobDefaultPriority)
+	}
+	if b.JobMaxPriority != nil {
+		result.JobMaxPriority = pointer.Of(*b.JobMaxPriority)
+	}
 	if b.EvalGCThreshold != "" {
 		result.EvalGCThreshold = b.EvalGCThreshold
 	}

command/agent/config_parse_test.go

@@ -147,7 +147,9 @@ var basicConfig = &Config{
 				ServiceSchedulerEnabled: true,
 			},
 		},
-		LicensePath: "/tmp/nomad.hclic",
+		LicensePath:        "/tmp/nomad.hclic",
+		JobDefaultPriority: pointer.Of(100),
+		JobMaxPriority:     pointer.Of(200),
 	},
 	ACL: &ACLConfig{
 		Enabled:                  true,

command/agent/config_test.go

@@ -358,6 +358,8 @@ func TestConfig_Merge(t *testing.T) {
 				NodeThreshold: 100,
 				NodeWindow:    11 * time.Minute,
 			},
+			JobMaxPriority:     pointer.Of(200),
+			JobDefaultPriority: pointer.Of(100),
 		},
 		ACL: &ACLConfig{
 			Enabled:               true,

command/agent/testdata/basic.hcl

@@ -133,6 +133,8 @@ server {
   raft_multiplier               = 4
   enable_event_broker           = false
   event_buffer_size             = 200
+  job_default_priority          = 100
+  job_max_priority              = 200
 
   plan_rejection_tracker {
     enabled        = true

command/agent/testdata/basic.json

@@ -320,7 +320,9 @@
         }]
       }],
       "upgrade_version": "0.8.0",
-      "license_path": "/tmp/nomad.hclic"
+      "license_path": "/tmp/nomad.hclic",
+      "job_default_priority": 100,
+      "job_max_priority": 200
     }
   ],
   "syslog_facility": "LOCAL1",

nomad/config.go

@@ -412,6 +412,12 @@ type Config struct {
 	// DeploymentQueryRateLimit is in queries per second and is used by the
 	// DeploymentWatcher to throttle the amount of simultaneously deployments
 	DeploymentQueryRateLimit float64
+
+	// JobDefaultPriority is the default Job priority if not specified.
+	JobDefaultPriority int
+
+	// JobMaxPriority is an upper bound on the Job priority.
+	JobMaxPriority int
 }
 
 func (c *Config) Copy() *Config {
@@ -529,6 +535,8 @@ func DefaultConfig() *Config {
 			},
 		},
 		DeploymentQueryRateLimit: deploymentwatcher.LimitStateQueriesPerSecond,
+		JobDefaultPriority:       structs.JobDefaultPriority,
+		JobMaxPriority:           structs.JobDefaultMaxPriority,
 	}
 
 	// Enable all known schedulers by default

nomad/job_endpoint.go

@@ -76,7 +76,7 @@ func NewJobEndpoints(s *Server, ctx *RPCContext) *Job {
 			jobExposeCheckHook{},
 			jobVaultHook{srv: s},
 			jobNamespaceConstraintCheckHook{srv: s},
-			jobValidate{},
+			&jobValidate{srv: s},
 			&memoryOversubscriptionValidate{srv: s},
 		},
 	}
@@ -990,7 +990,7 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st
 			continue
 		}
 
-		priority := structs.JobDefaultPriority
+		priority := j.srv.config.JobDefaultPriority
 		jtype := structs.JobTypeService
 		if job != nil {
 			priority = job.Priority

nomad/job_endpoint_hooks.go

@@ -253,13 +253,15 @@ func mutateConstraint(matcher constraintMatcher, taskGroup *structs.TaskGroup, c
 // jobValidate validates a Job and task drivers and returns an error if there is
 // a validation problem or if the Job is of a type a user is not allowed to
 // submit.
-type jobValidate struct{}
+type jobValidate struct {
+	srv *Server
+}
 
-func (jobValidate) Name() string {
+func (*jobValidate) Name() string {
 	return "validate"
 }
 
-func (jobValidate) Validate(job *structs.Job) (warnings []error, err error) {
+func (v *jobValidate) Validate(job *structs.Job) (warnings []error, err error) {
 	validationErrors := new(multierror.Error)
 	if err := job.Validate(); err != nil {
 		multierror.Append(validationErrors, err)
@@ -287,6 +289,10 @@ func (jobValidate) Validate(job *structs.Job) (warnings []error, err error) {
 		multierror.Append(validationErrors, fmt.Errorf("job can't be submitted with a payload, only dispatched"))
 	}
 
+	if job.Priority < structs.JobMinPriority || job.Priority > v.srv.config.JobMaxPriority {
+		multierror.Append(validationErrors, fmt.Errorf("job priority must be between [%d, %d]", structs.JobMinPriority, v.srv.config.JobMaxPriority))
+	}
+
 	return warnings, validationErrors.ErrorOrNil()
 }