inject: Configure proxy stream lifetime limits (#11837)

linkerd/linkerd2-proxy#2587 adds configuration parameters that bound the
lifetime and idle times of control plane streams. This change helps to
mitigate imbalanced control plane replica usage and to generally prevent
scenarios where a stream becomes "stuck," as has been observed when a
control plane replica is unhealthy.

This change adds helm values to control this behavior. Default values
are provided.

charts/linkerd-control-plane/README.md

@@ -231,6 +231,9 @@ Kubernetes: `>=1.22.0-0`
 | profileValidator.namespaceSelector | object | `{"matchExpressions":[{"key":"config.linkerd.io/admission-webhooks","operator":"NotIn","values":["disabled"]}]}` | Namespace selector used by admission webhook |
 | prometheusUrl | string | `""` | url of external prometheus instance (used for the heartbeat) |
 | proxy.await | bool | `true` | If set, the application container will not start until the proxy is ready |
+| proxy.control.streams.idleTimeout | string | `"5m"` | The timeout between consecutive updates from the control plane. |
+| proxy.control.streams.initialTimeout | string | `"3s"` | The timeout for the first update from the control plane. |
+| proxy.control.streams.lifetime | string | `"1h"` | The maximum duration for a response stream (i.e. before it will be reinitialized). |
 | proxy.cores | int | `0` | The `cpu.limit` and `cores` should be kept in sync. The value of `cores` must be an integer and should typically be set by rounding up from the limit. E.g. if cpu.limit is '1500m', cores should be 2. |
 | proxy.defaultInboundPolicy | string | "all-unauthenticated" | The default allow policy to use when no `Server` selects a pod.  One of: "all-authenticated", "all-unauthenticated", "cluster-authenticated", "cluster-unauthenticated", "deny" |
 | proxy.disableInboundProtocolDetectTimeout | bool | `false` | When set to true, disables the protocol detection timeout on the inbound side of the proxy by setting it to a very high value |

charts/linkerd-control-plane/values.yaml

@@ -207,6 +207,17 @@ proxy:
     initialDelaySeconds: 0
     periodSeconds: 1
     failureThreshold: 120
+  # Configures general properties of the proxy's control plane clients.
+  control:
+    # Configures limits on API response streams.
+    streams:
+      # -- The timeout for the first update from the control plane.
+      initialTimeout: "3s"
+      # -- The timeout between consecutive updates from the control plane.
+      idleTimeout: "5m"
+      # -- The maximum duration for a response stream (i.e. before it will be
+      # reinitialized).
+      lifetime: "1h"
 
 # proxy-init configuration
 proxyInit:

charts/partials/templates/_proxy.tpl

@@ -44,6 +44,12 @@ env:
   value: {{.Values.proxy.defaultInboundPolicy}}
 - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
   value: {{.Values.clusterNetworks | quote}}
+- name: LINKERD2_PROXY_CONTROL_STREAM_INITIAL_TIMEOUT
+  value: {{((.Values.proxy.control).streams).initialTimeout | default "" | quote}}
+- name: LINKERD2_PROXY_CONTROL_STREAM_IDLE_TIMEOUT
+  value: {{((.Values.proxy.control).streams).idleTimeout | default "" | quote}}
+- name: LINKERD2_PROXY_CONTROL_STREAM_LIFETIME
+  value: {{((.Values.proxy.control).streams).lifetime | default "" | quote}}
 {{ if .Values.proxy.inboundConnectTimeout -}}
 - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
   value: {{.Values.proxy.inboundConnectTimeout | quote}}

cli/cmd/inject_test.go

@@ -33,6 +33,8 @@ func mkFilename(filename string, verbose bool) string {
 }
 
 func testUninjectAndInject(t *testing.T, tc testCase) {
+	t.Helper()
+
 	file, err := os.Open("testdata/" + tc.inputFileName)
 	if err != nil {
 		t.Errorf("error opening test input file: %v\n", err)

cli/cmd/testdata/inject-filepath/expected/injected_nginx.yaml

@@ -48,6 +48,12 @@ spec:
           value: all-unauthenticated
         - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
           value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_CONTROL_STREAM_INITIAL_TIMEOUT
+          value: 3s
+        - name: LINKERD2_PROXY_CONTROL_STREAM_IDLE_TIMEOUT
+          value: 5m
+        - name: LINKERD2_PROXY_CONTROL_STREAM_LIFETIME
+          value: 1h
         - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
           value: 100ms
         - name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT

cli/cmd/testdata/inject-filepath/expected/injected_nginx_redis.yaml

@@ -48,6 +48,12 @@ spec:
           value: all-unauthenticated
         - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
           value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_CONTROL_STREAM_INITIAL_TIMEOUT
+          value: 3s
+        - name: LINKERD2_PROXY_CONTROL_STREAM_IDLE_TIMEOUT
+          value: 5m
+        - name: LINKERD2_PROXY_CONTROL_STREAM_LIFETIME
+          value: 1h
         - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
           value: 100ms
         - name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT
@@ -262,6 +268,12 @@ spec:
           value: all-unauthenticated
         - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
           value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_CONTROL_STREAM_INITIAL_TIMEOUT
+          value: 3s
+        - name: LINKERD2_PROXY_CONTROL_STREAM_IDLE_TIMEOUT
+          value: 5m
+        - name: LINKERD2_PROXY_CONTROL_STREAM_LIFETIME
+          value: 1h
         - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
           value: 100ms
         - name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT

cli/cmd/testdata/inject-filepath/expected/injected_redis.yaml

@@ -48,6 +48,12 @@ spec:
           value: all-unauthenticated
         - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
           value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_CONTROL_STREAM_INITIAL_TIMEOUT
+          value: 3s
+        - name: LINKERD2_PROXY_CONTROL_STREAM_IDLE_TIMEOUT
+          value: 5m
+        - name: LINKERD2_PROXY_CONTROL_STREAM_LIFETIME
+          value: 1h
         - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
           value: 100ms
         - name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT