Enable Envoy healthchecks

Signed-off-by: Steve Sloka <[email protected]>
projectcontour · Oct 18, 2018 · 193a4ce · 193a4ce
1 parent 040fa01
commit 193a4ce
Show file tree

Hide file tree

Showing 11 changed files with 64 additions and 83 deletions.
diff --git a/cmd/contour/contour.go b/cmd/contour/contour.go
@@ -23,23 +23,20 @@ import (
 	"strings"
 
 	clientset "github.com/heptio/contour/apis/generated/clientset/versioned"
+	"github.com/heptio/contour/internal/contour"
 	"github.com/heptio/contour/internal/debug"
+	"github.com/heptio/contour/internal/envoy"
+	"github.com/heptio/contour/internal/grpc"
 	"github.com/heptio/contour/internal/httpsvc"
+	"github.com/heptio/contour/internal/k8s"
+	"github.com/heptio/contour/internal/metrics"
 	"github.com/heptio/workgroup"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
-
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-
-	"github.com/heptio/contour/internal/contour"
-	"github.com/heptio/contour/internal/envoy"
-	"github.com/heptio/contour/internal/grpc"
-	"github.com/heptio/contour/internal/k8s"
-	"github.com/heptio/contour/internal/metrics"
-
-	"github.com/sirupsen/logrus"
 )
 
 var ingressrouteRootNamespaceFlag string

diff --git a/deployment/deployment-grpc-v2/02-contour.yaml b/deployment/deployment-grpc-v2/02-contour.yaml
@@ -40,6 +40,12 @@ spec:
         - --service-node node0
         - --log-level info
         - --v2-config-only
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
         - name: contour-config
           mountPath: /config
@@ -50,7 +56,7 @@ spec:
         command: ["contour"]
         args:
         - bootstrap
-        # Uncomment the statsd-enable to enable prometheus metrics
+        # Uncomment the statsd-enable to enable statsd metrics
         #- --statsd-enable
         # Uncomment to set a custom stats emission address and port
         #- --stats-address=0.0.0.0

diff --git a/deployment/ds-grpc-v2/02-contour.yaml b/deployment/ds-grpc-v2/02-contour.yaml
@@ -41,6 +41,12 @@ spec:
         - --service-node node0
         - --log-level info
         - --v2-config-only
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
         - name: contour-config
           mountPath: /config

diff --git a/deployment/ds-hostnet-split/03-envoy.yaml b/deployment/ds-hostnet-split/03-envoy.yaml
@@ -45,6 +45,12 @@ spec:
           hostPort: 443
           name: https
           protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
           - name: contour-config
             mountPath: /config
@@ -58,7 +64,6 @@ spec:
         - $(CONTOUR_SERVICE_HOST)
         - --xds-port
         - $(CONTOUR_SERVICE_PORT)
-        - --statsd-enabled
         command:
         - contour
         image: gcr.io/heptio-images/contour:v0.6.0-beta.3

diff --git a/deployment/ds-hostnet/02-contour.yaml b/deployment/ds-hostnet/02-contour.yaml
@@ -45,6 +45,12 @@ spec:
         - --service-node node0
         - --log-level info
         - --v2-config-only
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
         - name: contour-config
           mountPath: /config

diff --git a/deployment/render/daemonset-rbac.yaml b/deployment/render/daemonset-rbac.yaml
@@ -196,6 +196,12 @@ spec:
         - --service-node node0
         - --log-level info
         - --v2-config-only
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
         - name: contour-config
           mountPath: /config

diff --git a/deployment/render/deployment-rbac.yaml b/deployment/render/deployment-rbac.yaml
@@ -195,6 +195,12 @@ spec:
         - --service-node node0
         - --log-level info
         - --v2-config-only
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8002
+          initialDelaySeconds: 3
+          periodSeconds: 3
         volumeMounts:
         - name: contour-config
           mountPath: /config
@@ -205,7 +211,7 @@ spec:
         command: ["contour"]
         args:
         - bootstrap
-        # Uncomment the statsd-enable to enable prometheus metrics
+        # Uncomment the statsd-enable to enable statsd metrics
         #- --statsd-enable
         # Uncomment to set a custom stats emission address and port
         #- --stats-address=0.0.0.0

diff --git a/docs/architecture.md b/docs/architecture.md
@@ -19,4 +19,7 @@ Contour is a client of the Kubernetes API. Contour watches Ingress, Service, and
 The transfer of information from Kubernetes to Contour is by watching the API with the SharedInformer framework.
 The transfer of information from Contour to Envoy is by polling from the Envoy side.
 
+Kubernetes Readiness Probes are configured to check the status of Envoy.
+These are enabled over the metrics port and are served over http via `/healthz`.
+
 [0]: https://github.com/envoyproxy/data-plane-api#terminology
diff --git a/docs/prometheus.md b/docs/prometheus.md
@@ -9,8 +9,8 @@ avoid exposing the entire admin interface to Prometheus (and other workloads in
 the cluster), Contour configures a static listener that sends traffic to the
 stats endpoint and nowhere else.
 
-To enable the static listener, set the `--statsd-enabled` flag on the Contour
-`bootstrap` command that runs as an init container.
+To enable the static listener, set the `--statsd-enabled` flag.
+By default, Envoy's stats will be exposed over `0.0.0.0:8002` but can be overridden setting the `--stats-address` and `--stats-port` flags in Contour.
 
 ### Configuration Prometheus
 

diff --git a/internal/envoy/config.go b/internal/envoy/config.go
@@ -117,7 +117,7 @@ static_resources:
           protocol: TCP
           address: 127.0.0.1
           port_value: {{ if .AdminPort }}{{ .AdminPort }}{{ else }}9001{{ end }}
-{{ if .StatsdEnabled }}  listeners:
+  listeners:
     - address:
         socket_address:
           protocol: TCP
@@ -140,18 +140,21 @@ static_resources:
                           route:
                             cluster: service_stats
                 http_filters:
+                  - name: envoy.health_check
+                    config:
+                      endpoint: "/healthz"
+                      pass_through_mode: false
                   - name: envoy.router
                     config:
-stats_sinks:
+{{ if .StatsdEnabled }}stats_sinks:
   - name: envoy.statsd
     config:
       address:
         socket_address:
           protocol: UDP
           address: {{ if .StatsdAddress }}{{ .StatsdAddress }}{{ else }}127.0.0.1{{ end }}
           port_value: {{ if .StatsdPort }}{{ .StatsdPort }}{{ else }}9125{{ end }}
-{{ end -}}
-admin:
+{{ end -}}admin:
   access_log_path: {{ if .AdminAccessLogPath }}{{ .AdminAccessLogPath }}{{ else }}/dev/null{{ end }}
   address:
     socket_address:

diff --git a/internal/envoy/config_test.go b/internal/envoy/config_test.go
@@ -38,63 +38,6 @@ func TestConfigWriter_WriteYAML(t *testing.T) {
       grpc_services:
       - envoy_grpc:
           cluster_name: contour
-static_resources:
-  clusters:
-  - name: contour
-    connect_timeout: { seconds: 5 }
-    type: STRICT_DNS
-    hosts:
-    - socket_address:
-        address: 127.0.0.1
-        port_value: 8001
-    lb_policy: ROUND_ROBIN
-    http2_protocol_options: {}
-    circuit_breakers:
-      thresholds:
-        - priority: high
-          max_connections: 100000
-          max_pending_requests: 100000
-          max_requests: 60000000
-          max_retries: 50
-        - priority: default
-          max_connections: 100000
-          max_pending_requests: 100000
-          max_requests: 60000000
-          max_retries: 50
-  - name: service_stats
-    connect_timeout: 0.250s
-    type: LOGICAL_DNS
-    lb_policy: ROUND_ROBIN
-    hosts:
-      - socket_address:
-          protocol: TCP
-          address: 127.0.0.1
-          port_value: 9001
-admin:
-  access_log_path: /dev/null
-  address:
-    socket_address:
-      address: 127.0.0.1
-      port_value: 9001
-`,
-		},
-		"statsd enabled": {
-			ConfigWriter: ConfigWriter{
-				StatsdEnabled: true,
-			},
-			want: `dynamic_resources:
-  lds_config:
-    api_config_source:
-      api_type: GRPC
-      grpc_services:
-      - envoy_grpc:
-          cluster_name: contour
-  cds_config:
-    api_config_source:
-      api_type: GRPC
-      grpc_services:
-      - envoy_grpc:
-          cluster_name: contour
 static_resources:
   clusters:
   - name: contour
@@ -150,16 +93,12 @@ static_resources:
                           route:
                             cluster: service_stats
                 http_filters:
+                  - name: envoy.health_check
+                    config:
+                      endpoint: "/healthz"
+                      pass_through_mode: false
                   - name: envoy.router
                     config:
-stats_sinks:
-  - name: envoy.statsd
-    config:
-      address:
-        socket_address:
-          protocol: UDP
-          address: 127.0.0.1
-          port_value: 9125
 admin:
   access_log_path: /dev/null
   address:
@@ -242,6 +181,10 @@ static_resources:
                           route:
                             cluster: service_stats
                 http_filters:
+                  - name: envoy.health_check
+                    config:
+                      endpoint: "/healthz"
+                      pass_through_mode: false
                   - name: envoy.router
                     config:
 stats_sinks: