diff --git a/CHANGELOG.md b/CHANGELOG.md index e6aa48f588d..f86fc5aceed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ IMPROVEMENTS: * cli: Cross-namespace `nomad job` commands will now select exact matches if the selection is unambiguous. [[GH-10648](https://github.com/hashicorp/nomad/issues/10648)] * client/fingerprint: Consul fingerprinter probes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)] * csi: Validate that `volume` blocks for CSI volumes include the required `attachment_mode` and `access_mode` fields. [[GH-10651](https://github.com/hashicorp/nomad/issues/10651)] +* server: Make deployment rate limiting configurable for high volume loads [[GH-10706](https://github.com/hashicorp/nomad/pull/10706)] BUG FIXES: * api: Fixed event stream connection initialization when there are no events to send [[GH-10637](https://github.com/hashicorp/nomad/issues/10637)] diff --git a/command/agent/agent.go b/command/agent/agent.go index 18cd812c3b2..ff20dc99ab4 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -29,6 +29,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad" + "github.com/hashicorp/nomad/nomad/deploymentwatcher" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/raft" @@ -420,6 +421,15 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { conf.RPCMaxConnsPerClient = limit } + // Set deployment rate limit + if rate := agentConfig.Server.DeploymentQueryRateLimit; rate == 0 { + conf.DeploymentQueryRateLimit = deploymentwatcher.LimitStateQueriesPerSecond + } else if rate > 0 { + conf.DeploymentQueryRateLimit = rate + } else { + return nil, fmt.Errorf("deploy_query_rate_limit must be greater than 0") + } + // Add Enterprise license configs conf.LicenseEnv = agentConfig.Server.LicenseEnv conf.LicensePath = agentConfig.Server.LicensePath diff --git a/command/agent/config.go b/command/agent/config.go index c973c573831..02fdb55880b 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -516,6 +516,10 @@ type ServerConfig struct { ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` Search *Search `hcl:"search"` + + // DeploymentQueryRateLimit is in queries per second and is used by the + // DeploymentWatcher to throttle the amount of simultaneously deployments + DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"` } // Search is used in servers to configure search API options. @@ -1522,6 +1526,10 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { result.DefaultSchedulerConfig = &c } + if b.DeploymentQueryRateLimit != 0 { + result.DeploymentQueryRateLimit = b.DeploymentQueryRateLimit + } + if b.Search != nil { result.Search = &Search{FuzzyEnabled: b.Search.FuzzyEnabled} if b.Search.LimitQuery > 0 { diff --git a/nomad/config.go b/nomad/config.go index 05db46fb383..8c1f38a9117 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/deploymentwatcher" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/scheduler" @@ -363,6 +364,10 @@ type Config struct { // AgentShutdown is used to call agent.Shutdown from the context of a Server // It is used primarily for licensing AgentShutdown func() error + + // DeploymentQueryRateLimit is in queries per second and is used by the + // DeploymentWatcher to throttle the amount of simultaneously deployments + DeploymentQueryRateLimit float64 } // CheckVersion is used to check if the ProtocolVersion is valid @@ -448,6 +453,7 @@ func DefaultConfig() *Config { ServiceSchedulerEnabled: false, }, }, + DeploymentQueryRateLimit: deploymentwatcher.LimitStateQueriesPerSecond, } // Enable all known schedulers by default diff --git a/nomad/server.go b/nomad/server.go index 4c55b3c8f62..91d42f29e6c 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1021,7 +1021,7 @@ func (s *Server) setupDeploymentWatcher() error { raftShim, s.staticEndpoints.Deployment, s.staticEndpoints.Job, - deploymentwatcher.LimitStateQueriesPerSecond, + s.config.DeploymentQueryRateLimit, deploymentwatcher.CrossDeploymentUpdateBatchDuration, )