Skip to content

Commit

Permalink
consul/connect: fix bug causing high cpu with multiple connect sideca…
Browse files Browse the repository at this point in the history 
…rs in group

This PR fixes a bug where the underlying Envoy process of a Connect gateway
would consume a full core of CPU if there is more than one sidecar or gateway
in a group. The utilization was being caused by Consul injecting an envoy_ready_listener
on 127.0.0.1:8443, of which only one of the Envoys would be able to bind to.
The others would spin in a hot loop trying to bind the listener.

As a workaround, we now specify -address during the Envoy bootstrap config
step, which is how Consul maps this ready listener. Because there is already
the envoy_admin_listener, and we need to continue supporting running gateways
in host networking mode, and in those case we want to use the same port
value coming from the service.port field, we now bind the admin listener to
the 127.0.0.2 loop-back interface, and the ready listener takes 127.0.0.1.

This shouldn't make a difference in the 99.999% use case where envoy is
being run in its official docker container. Advanced users can reference
${NOMAD_ENVOY_ADMIN_ADDR_<service>} (as they 'ought to) if needed,
as well as the new variable ${NOMAD_ENVOY_READY_ADDR_<service>} for the
envoy_ready_listener.
  • Loading branch information
@shoenig
shoenig committed Jul 9, 2021
1 parent c4e0355 commit ee7d32f
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 33 deletions.
3 changes: 3 additions & 0 deletions .changelog/10883.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
consul/connect: Fixed a bug causing high CPU with multiple connect sidecars in one group
```
59 changes: 41 additions & 18 deletions client/allocrunner/taskrunner/envoy_bootstrap_hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,10 @@ func newEnvoyBootstrapHookConfig(alloc *structs.Allocation, consul *config.Consu
}

const (
envoyBaseAdminPort = 19000
envoyBaseAdminPort = 19000 // Consul default (bridge only)
envoyBaseReadyPort = 19100 // Consul default (bridge only)
envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_"
envoyReadyBindEnvPrefix = "NOMAD_ENVOY_READY_ADDR_"
)

const (
Expand Down Expand Up @@ -240,12 +242,17 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart

h.logger.Debug("bootstrapping Consul "+serviceKind, "task", req.Task.Name, "service", serviceName)

// Envoy runs an administrative API on the loopback interface. There is no
// way to turn this feature off.
// Envoy runs an administrative listener. There is no way to turn this feature off.
// https://github.com/envoyproxy/envoy/issues/1297
envoyAdminBind := buildEnvoyAdminBind(h.alloc, serviceName, req.Task.Name, req.TaskEnv)

// Consul configures a ready listener. There is no way to turn this feature off.
envoyReadyBind := buildEnvoyReadyBind(h.alloc, serviceName, req.Task.Name, req.TaskEnv)

// Set runtime environment variables for the envoy admin and ready listeners.
resp.Env = map[string]string{
helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind,
helper.CleanEnvVar(envoyReadyBindEnvPrefix+serviceName, '_'): envoyReadyBind,
}

// Envoy bootstrap configuration may contain a Consul token, so write
Expand All @@ -259,7 +266,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
}
h.logger.Debug("check for SI token for task", "task", req.Task.Name, "exists", siToken != "")

bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, siToken, bootstrapFilePath)
bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, envoyReadyBind, siToken, bootstrapFilePath)
bootstrapArgs := bootstrap.args()
bootstrapEnv := bootstrap.env(os.Environ())

Expand Down Expand Up @@ -331,37 +338,50 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
return nil
}

// buildEnvoyAdminBind determines a unique port for use by the envoy admin
// listener.
// buildEnvoyAdminBind determines a unique port for use by the envoy admin listener.
//
// This listener will be bound to 127.0.0.2.
func buildEnvoyAdminBind(alloc *structs.Allocation, service, task string, env *taskenv.TaskEnv) string {
return buildEnvoyBind(alloc, "127.0.0.2", service, task, env, envoyBaseAdminPort)
}

// buildEnvoyAdminBind determines a unique port for use by the envoy ready listener.
//
// This listener will be bound to 127.0.0.1.
func buildEnvoyReadyBind(alloc *structs.Allocation, service, task string, env *taskenv.TaskEnv) string {
return buildEnvoyBind(alloc, "127.0.0.1", service, task, env, envoyBaseReadyPort)
}

// buildEnvoyBind is used to determine a unique port for an envoy listener.
//
// In bridge mode, if multiple sidecars are running, the bind addresses need
// to be unique within the namespace, so we simply start at 19000 and increment
// to be unique within the namespace, so we simply start at basePort and increment
// by the index of the task.
//
// In host mode, use the port provided through the service definition, which can
// be a port chosen by Nomad.
func buildEnvoyAdminBind(alloc *structs.Allocation, serviceName, taskName string, taskEnv *taskenv.TaskEnv) string {
func buildEnvoyBind(alloc *structs.Allocation, ifce, service, task string, taskEnv *taskenv.TaskEnv, basePort int) string {
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
port := envoyBaseAdminPort
port := basePort
switch tg.Networks[0].Mode {
case "host":
interpolatedServices := taskenv.InterpolateServices(taskEnv, tg.Services)
for _, service := range interpolatedServices {
if service.Name == serviceName {
mapping := tg.Networks.Port(service.PortLabel)
for _, svc := range interpolatedServices {
if svc.Name == service {
mapping := tg.Networks.Port(svc.PortLabel)
port = mapping.Value
break
}
}
default:
for idx, task := range tg.Tasks {
if task.Name == taskName {
for idx, tgTask := range tg.Tasks {
if tgTask.Name == task {
port += idx
break
}
}
}
return fmt.Sprintf("localhost:%d", port)
return fmt.Sprintf("%s:%d", ifce, port)
}

func (h *envoyBootstrapHook) writeConfig(filename, config string) error {
Expand Down Expand Up @@ -421,7 +441,7 @@ func (h *envoyBootstrapHook) proxyServiceID(group string, service *structs.Servi
// https://www.consul.io/commands/connect/envoy#consul-connect-envoy
func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
group string, service *structs.Service,
grpcAddr, envoyAdminBind, siToken, filepath string,
grpcAddr, envoyAdminBind, envoyReadyBind, siToken, filepath string,
) envoyBootstrapArgs {
var (
sidecarForID string // sidecar only
Expand Down Expand Up @@ -450,15 +470,16 @@ func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
h.logger.Info("bootstrapping envoy",
"sidecar_for", service.Name, "bootstrap_file", filepath,
"sidecar_for_id", sidecarForID, "grpc_addr", grpcAddr,
"admin_bind", envoyAdminBind, "gateway", gateway,
"proxy_id", proxyID, "namespace", namespace,
"admin_bind", envoyAdminBind, "ready_bind", envoyReadyBind,
"gateway", gateway, "proxy_id", proxyID, "namespace", namespace,
)

return envoyBootstrapArgs{
consulConfig: h.consulConfig,
sidecarFor: sidecarForID,
grpcAddr: grpcAddr,
envoyAdminBind: envoyAdminBind,
envoyReadyBind: envoyReadyBind,
siToken: siToken,
gateway: gateway,
proxyID: proxyID,
Expand All @@ -474,6 +495,7 @@ type envoyBootstrapArgs struct {
sidecarFor string // sidecars only
grpcAddr string
envoyAdminBind string
envoyReadyBind string
siToken string
gateway string // gateways only
proxyID string // gateways only
Expand All @@ -489,6 +511,7 @@ func (e envoyBootstrapArgs) args() []string {
"-grpc-addr", e.grpcAddr,
"-http-addr", e.consulConfig.HTTPAddr,
"-admin-bind", e.envoyAdminBind,
"-address", e.envoyReadyBind,
"-bootstrap",
}

Expand Down
32 changes: 21 additions & 11 deletions client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,15 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulPlainConfig,
envoyAdminBind: "localhost:3333",
envoyAdminBind: "127.0.0.2:19000",
envoyReadyBind: "127.0.0.1:19100",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-admin-bind", "127.0.0.2:19000",
"-address", "127.0.0.1:19100",
"-bootstrap",
"-sidecar-for", "s1",
}, result)
Expand All @@ -141,14 +143,16 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulPlainConfig,
envoyAdminBind: "localhost:3333",
envoyAdminBind: "127.0.0.2:19000",
envoyReadyBind: "127.0.0.1:19100",
siToken: token,
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-admin-bind", "127.0.0.2:19000",
"-address", "127.0.0.1:19100",
"-bootstrap",
"-sidecar-for", "s1",
"-token", token,
Expand All @@ -160,13 +164,15 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulTLSConfig,
envoyAdminBind: "localhost:3333",
envoyAdminBind: "127.0.0.2:19000",
envoyReadyBind: "127.0.0.1:19100",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-admin-bind", "127.0.0.2:19000",
"-address", "127.0.0.1:19100",
"-bootstrap",
"-sidecar-for", "s1",
"-ca-file", "/etc/tls/ca-file",
Expand All @@ -179,15 +185,17 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
ebArgs := envoyBootstrapArgs{
consulConfig: consulPlainConfig,
grpcAddr: "1.1.1.1",
envoyAdminBind: "localhost:3333",
envoyAdminBind: "127.0.0.2:19000",
envoyReadyBind: "127.0.0.1:19100",
gateway: "my-ingress-gateway",
proxyID: "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-admin-bind", "127.0.0.2:19000",
"-address", "127.0.0.1:19100",
"-bootstrap",
"-gateway", "my-ingress-gateway",
"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
Expand All @@ -198,15 +206,17 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
ebArgs := envoyBootstrapArgs{
consulConfig: consulPlainConfig,
grpcAddr: "1.1.1.1",
envoyAdminBind: "localhost:3333",
envoyAdminBind: "127.0.0.2:19000",
envoyReadyBind: "127.0.0.1:19100",
gateway: "my-mesh-gateway",
proxyID: "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-admin-bind", "127.0.0.2:19000",
"-address", "127.0.0.1:19100",
"-bootstrap",
"-gateway", "my-mesh-gateway",
"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
Expand Down Expand Up @@ -453,7 +463,7 @@ func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
require.True(t, resp.Done)

require.NotNil(t, resp.Env)
require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
require.Equal(t, "127.0.0.2:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])

// Ensure the default path matches
env := map[string]string{
Expand Down
20 changes: 16 additions & 4 deletions website/content/partials/envvars.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,11 +264,23 @@
<code>NOMAD_ENVOY_ADMIN_ADDR_&lt;service&gt;</code>
</td>
<td>
Local address <code>localhost:Port</code> for the admin port of the
Local address <code>127.0.0.2:Port</code> for the admin port of the
envoy sidecar for the given <code>service</code> when defined as a
Consul Connect enabled service.
</td>
</tr>
Consul Connect enabled service. Envoy runs inside the group network
namespace unless configured for host networking.
</td>
</tr>
<tr>
<td>
<code>NOMAD_ENVOY_READY_ADDR_&lt;service&gt;</code>
</td>
<td>
Local address <code>127.0.0.1:Port</code> for the ready port of the
envoy sidecar for the given <code>service</code> when defined as a
Consul Connect enabled service. Envoy runs inside the group network
namespace unless configured for host networking.
</td>
</tr>
<tr>
<th colspan="2">Consul-related Variables (only set for connect native tasks)</th>
</tr>
Expand Down

0 comments on commit ee7d32f

Please sign in to comment.