From 3fa44979a6115325ae76f7fbc658faaacc966f43 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 10:24:12 -0700 Subject: [PATCH 1/7] deployment-watcher: add config plumbing for a deployment-rate-limit --- command/agent/agent.go | 10 ++++++++++ command/agent/command.go | 1 + command/agent/config.go | 4 ++++ nomad/config.go | 4 ++++ nomad/server.go | 2 +- 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 18cd812c3b2..235dd40873f 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -3,6 +3,7 @@ package agent import ( "context" "fmt" + "github.com/hashicorp/nomad/nomad/deploymentwatcher" "io" "io/ioutil" golog "log" @@ -420,6 +421,15 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { conf.RPCMaxConnsPerClient = limit } + // Set deployment rate limit + if rate := agentConfig.Server.DeploymentRateLimit; rate == 0 { + conf.DeploymentRateLimit = deploymentwatcher.LimitStateQueriesPerSecond + } else if rate > 0 { + conf.DeploymentRateLimit = rate + } else { + return nil, fmt.Errorf("deployment-rate-limit must be greater than 0") + } + // Add Enterprise license configs conf.LicenseEnv = agentConfig.Server.LicenseEnv conf.LicensePath = agentConfig.Server.LicensePath diff --git a/command/agent/command.go b/command/agent/command.go index 0346a726c14..d8b70a2930f 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -98,6 +98,7 @@ func (c *Command) readConfig() *Config { cmdConfig.Server.ServerJoin.RetryInterval = d return nil }), "retry-interval", "") + flags.Float64Var(&cmdConfig.Server.DeploymentRateLimit, "deployment-rate-limit", 0, "") // Client-only options flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "") diff --git a/command/agent/config.go b/command/agent/config.go index c973c573831..0e87aade25a 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -516,6 +516,10 @@ type ServerConfig struct { ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` Search *Search `hcl:"search"` + + // DeploymentRateLimit is in queries per second and is used by the + // DeploymentWatcher to throttle the amount of simultaneously deployments + DeploymentRateLimit float64 } // Search is used in servers to configure search API options. diff --git a/nomad/config.go b/nomad/config.go index 05db46fb383..5b29d7cc059 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -363,6 +363,10 @@ type Config struct { // AgentShutdown is used to call agent.Shutdown from the context of a Server // It is used primarily for licensing AgentShutdown func() error + + // DeploymentRateLimit is in queries per second and is used by the + // DeploymentWatcher to throttle the amount of simultaneously deployments + DeploymentRateLimit float64 } // CheckVersion is used to check if the ProtocolVersion is valid diff --git a/nomad/server.go b/nomad/server.go index 4c55b3c8f62..3a0b4541061 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1021,7 +1021,7 @@ func (s *Server) setupDeploymentWatcher() error { raftShim, s.staticEndpoints.Deployment, s.staticEndpoints.Job, - deploymentwatcher.LimitStateQueriesPerSecond, + s.config.DeploymentRateLimit, deploymentwatcher.CrossDeploymentUpdateBatchDuration, ) From 57bb4cf79fd843836ba85975bd5119aae5452dfe Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 10:34:48 -0700 Subject: [PATCH 2/7] changelog: configurable deployment rate limiting --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6aa48f588d..f86fc5aceed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ IMPROVEMENTS: * cli: Cross-namespace `nomad job` commands will now select exact matches if the selection is unambiguous. [[GH-10648](https://github.com/hashicorp/nomad/issues/10648)] * client/fingerprint: Consul fingerprinter probes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)] * csi: Validate that `volume` blocks for CSI volumes include the required `attachment_mode` and `access_mode` fields. [[GH-10651](https://github.com/hashicorp/nomad/issues/10651)] +* server: Make deployment rate limiting configurable for high volume loads [[GH-10706](https://github.com/hashicorp/nomad/pull/10706)] BUG FIXES: * api: Fixed event stream connection initialization when there are no events to send [[GH-10637](https://github.com/hashicorp/nomad/issues/10637)] From e40c70bf874ac3b21678704ff60d37820474a46a Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 10:37:17 -0700 Subject: [PATCH 3/7] Update command/agent/config.go Co-authored-by: Michael Schurter --- command/agent/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent/config.go b/command/agent/config.go index 0e87aade25a..33c9e8ca8ca 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -519,7 +519,7 @@ type ServerConfig struct { // DeploymentRateLimit is in queries per second and is used by the // DeploymentWatcher to throttle the amount of simultaneously deployments - DeploymentRateLimit float64 + DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limt"` } // Search is used in servers to configure search API options. From 62fbf5c8056c7c92852bd292609c050bd2617ea1 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 10:47:57 -0700 Subject: [PATCH 4/7] deployment-watcher: update query rate limit var name --- command/agent/agent.go | 8 ++++---- command/agent/command.go | 1 - command/agent/config.go | 4 ++-- nomad/config.go | 4 ++-- nomad/server.go | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 235dd40873f..3d643630fec 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -422,12 +422,12 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } // Set deployment rate limit - if rate := agentConfig.Server.DeploymentRateLimit; rate == 0 { - conf.DeploymentRateLimit = deploymentwatcher.LimitStateQueriesPerSecond + if rate := agentConfig.Server.DeploymentQueryRateLimit; rate == 0 { + conf.DeploymentQueryRateLimit = deploymentwatcher.LimitStateQueriesPerSecond } else if rate > 0 { - conf.DeploymentRateLimit = rate + conf.DeploymentQueryRateLimit = rate } else { - return nil, fmt.Errorf("deployment-rate-limit must be greater than 0") + return nil, fmt.Errorf("deployment_query_rate_limit must be greater than 0") } // Add Enterprise license configs diff --git a/command/agent/command.go b/command/agent/command.go index d8b70a2930f..0346a726c14 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -98,7 +98,6 @@ func (c *Command) readConfig() *Config { cmdConfig.Server.ServerJoin.RetryInterval = d return nil }), "retry-interval", "") - flags.Float64Var(&cmdConfig.Server.DeploymentRateLimit, "deployment-rate-limit", 0, "") // Client-only options flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "") diff --git a/command/agent/config.go b/command/agent/config.go index 33c9e8ca8ca..c2761fa3c7c 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -517,9 +517,9 @@ type ServerConfig struct { Search *Search `hcl:"search"` - // DeploymentRateLimit is in queries per second and is used by the + // DeploymentQueryRateLimit is in queries per second and is used by the // DeploymentWatcher to throttle the amount of simultaneously deployments - DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limt"` + DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"` } // Search is used in servers to configure search API options. diff --git a/nomad/config.go b/nomad/config.go index 5b29d7cc059..453b04e6267 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -364,9 +364,9 @@ type Config struct { // It is used primarily for licensing AgentShutdown func() error - // DeploymentRateLimit is in queries per second and is used by the + // DeploymentQueryRateLimit is in queries per second and is used by the // DeploymentWatcher to throttle the amount of simultaneously deployments - DeploymentRateLimit float64 + DeploymentQueryRateLimit float64 } // CheckVersion is used to check if the ProtocolVersion is valid diff --git a/nomad/server.go b/nomad/server.go index 3a0b4541061..91d42f29e6c 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1021,7 +1021,7 @@ func (s *Server) setupDeploymentWatcher() error { raftShim, s.staticEndpoints.Deployment, s.staticEndpoints.Job, - s.config.DeploymentRateLimit, + s.config.DeploymentQueryRateLimit, deploymentwatcher.CrossDeploymentUpdateBatchDuration, ) From 1d5b423d699cb57c45bfd16e7889f6b55a993205 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 11:24:20 -0700 Subject: [PATCH 5/7] add server config merge for new deploy rate limit value --- command/agent/agent.go | 2 +- command/agent/config.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 3d643630fec..d840d1c5ee0 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -427,7 +427,7 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } else if rate > 0 { conf.DeploymentQueryRateLimit = rate } else { - return nil, fmt.Errorf("deployment_query_rate_limit must be greater than 0") + return nil, fmt.Errorf("deploy_query_rate_limit must be greater than 0") } // Add Enterprise license configs diff --git a/command/agent/config.go b/command/agent/config.go index c2761fa3c7c..02fdb55880b 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -1526,6 +1526,10 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { result.DefaultSchedulerConfig = &c } + if b.DeploymentQueryRateLimit != 0 { + result.DeploymentQueryRateLimit = b.DeploymentQueryRateLimit + } + if b.Search != nil { result.Search = &Search{FuzzyEnabled: b.Search.FuzzyEnabled} if b.Search.LimitQuery > 0 { From c574fa52e74a9223adf436cf63257f5207a0d5e6 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 12:08:44 -0700 Subject: [PATCH 6/7] lint --- command/agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index d840d1c5ee0..ff20dc99ab4 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -3,7 +3,6 @@ package agent import ( "context" "fmt" - "github.com/hashicorp/nomad/nomad/deploymentwatcher" "io" "io/ioutil" golog "log" @@ -30,6 +29,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad" + "github.com/hashicorp/nomad/nomad/deploymentwatcher" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/raft" From f86526f68f85fded19e6946f0cb56545775f6df4 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Fri, 4 Jun 2021 12:17:56 -0700 Subject: [PATCH 7/7] add deployment query rate limit to default config --- nomad/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nomad/config.go b/nomad/config.go index 453b04e6267..8c1f38a9117 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/deploymentwatcher" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/scheduler" @@ -452,6 +453,7 @@ func DefaultConfig() *Config { ServiceSchedulerEnabled: false, }, }, + DeploymentQueryRateLimit: deploymentwatcher.LimitStateQueriesPerSecond, } // Enable all known schedulers by default