From 35859a73441a160ec08a44a61b4097ecc6b45876 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 16 Oct 2018 17:10:11 -0400 Subject: [PATCH] Enable Envoy healthchecks Signed-off-by: Steve Sloka --- cmd/contour/contour.go | 15 ++-- deployment/deployment-grpc-v2/02-contour.yaml | 8 +- deployment/ds-grpc-v2/02-contour.yaml | 6 ++ deployment/ds-hostnet-split/03-envoy.yaml | 7 +- deployment/ds-hostnet/02-contour.yaml | 6 ++ deployment/render/daemonset-rbac.yaml | 6 ++ deployment/render/deployment-rbac.yaml | 8 +- docs/architecture.md | 3 + docs/prometheus.md | 4 +- internal/envoy/config.go | 11 ++- internal/envoy/config_test.go | 73 ++----------------- 11 files changed, 62 insertions(+), 85 deletions(-) diff --git a/cmd/contour/contour.go b/cmd/contour/contour.go index 20c7b007655..93b272cb0f8 100644 --- a/cmd/contour/contour.go +++ b/cmd/contour/contour.go @@ -23,23 +23,20 @@ import ( "strings" clientset "github.com/heptio/contour/apis/generated/clientset/versioned" + "github.com/heptio/contour/internal/contour" "github.com/heptio/contour/internal/debug" + "github.com/heptio/contour/internal/envoy" + "github.com/heptio/contour/internal/grpc" "github.com/heptio/contour/internal/httpsvc" + "github.com/heptio/contour/internal/k8s" + "github.com/heptio/contour/internal/metrics" "github.com/heptio/workgroup" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" kingpin "gopkg.in/alecthomas/kingpin.v2" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - - "github.com/heptio/contour/internal/contour" - "github.com/heptio/contour/internal/envoy" - "github.com/heptio/contour/internal/grpc" - "github.com/heptio/contour/internal/k8s" - "github.com/heptio/contour/internal/metrics" - - "github.com/sirupsen/logrus" ) var ingressrouteRootNamespaceFlag string diff --git a/deployment/deployment-grpc-v2/02-contour.yaml b/deployment/deployment-grpc-v2/02-contour.yaml index 97323cd4098..4ed5931de87 100644 --- a/deployment/deployment-grpc-v2/02-contour.yaml +++ b/deployment/deployment-grpc-v2/02-contour.yaml @@ -40,6 +40,12 @@ spec: - --service-node node0 - --log-level info - --v2-config-only + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config @@ -50,8 +56,6 @@ spec: command: ["contour"] args: - bootstrap - # Uncomment the statsd-enable to enable prometheus metrics - #- --statsd-enable # Uncomment to set a custom stats emission address and port #- --stats-address=0.0.0.0 #- --stats-port=8002 diff --git a/deployment/ds-grpc-v2/02-contour.yaml b/deployment/ds-grpc-v2/02-contour.yaml index 0b479adf7e1..332c12068fa 100644 --- a/deployment/ds-grpc-v2/02-contour.yaml +++ b/deployment/ds-grpc-v2/02-contour.yaml @@ -41,6 +41,12 @@ spec: - --service-node node0 - --log-level info - --v2-config-only + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config diff --git a/deployment/ds-hostnet-split/03-envoy.yaml b/deployment/ds-hostnet-split/03-envoy.yaml index 26ce8f7569b..1b8042b0f9e 100644 --- a/deployment/ds-hostnet-split/03-envoy.yaml +++ b/deployment/ds-hostnet-split/03-envoy.yaml @@ -45,6 +45,12 @@ spec: hostPort: 443 name: https protocol: TCP + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config @@ -58,7 +64,6 @@ spec: - $(CONTOUR_SERVICE_HOST) - --xds-port - $(CONTOUR_SERVICE_PORT) - - --statsd-enabled command: - contour image: gcr.io/heptio-images/contour:v0.6.0-beta.3 diff --git a/deployment/ds-hostnet/02-contour.yaml b/deployment/ds-hostnet/02-contour.yaml index 67109ac8fc3..2b961f88cd9 100644 --- a/deployment/ds-hostnet/02-contour.yaml +++ b/deployment/ds-hostnet/02-contour.yaml @@ -45,6 +45,12 @@ spec: - --service-node node0 - --log-level info - --v2-config-only + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config diff --git a/deployment/render/daemonset-rbac.yaml b/deployment/render/daemonset-rbac.yaml index 6b19eba36d5..4d7166b6f6e 100644 --- a/deployment/render/daemonset-rbac.yaml +++ b/deployment/render/daemonset-rbac.yaml @@ -196,6 +196,12 @@ spec: - --service-node node0 - --log-level info - --v2-config-only + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config diff --git a/deployment/render/deployment-rbac.yaml b/deployment/render/deployment-rbac.yaml index a2ddcf65a8e..0a967178eec 100644 --- a/deployment/render/deployment-rbac.yaml +++ b/deployment/render/deployment-rbac.yaml @@ -195,6 +195,12 @@ spec: - --service-node node0 - --log-level info - --v2-config-only + readinessProbe: + httpGet: + path: /healthz + port: 8002 + initialDelaySeconds: 3 + periodSeconds: 3 volumeMounts: - name: contour-config mountPath: /config @@ -205,8 +211,6 @@ spec: command: ["contour"] args: - bootstrap - # Uncomment the statsd-enable to enable prometheus metrics - #- --statsd-enable # Uncomment to set a custom stats emission address and port #- --stats-address=0.0.0.0 #- --stats-port=8002 diff --git a/docs/architecture.md b/docs/architecture.md index 0c8d84ef458..64d7efcc74a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,4 +19,7 @@ Contour is a client of the Kubernetes API. Contour watches Ingress, Service, and The transfer of information from Kubernetes to Contour is by watching the API with the SharedInformer framework. The transfer of information from Contour to Envoy is by polling from the Envoy side. +Kubernetes Readiness Probes are configured to check the status of Envoy. +These are enabled over the metrics port and are served over http via `/healthz`. + [0]: https://github.com/envoyproxy/data-plane-api#terminology diff --git a/docs/prometheus.md b/docs/prometheus.md index 14e5057e101..fc8b319af29 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -9,8 +9,8 @@ avoid exposing the entire admin interface to Prometheus (and other workloads in the cluster), Contour configures a static listener that sends traffic to the stats endpoint and nowhere else. -To enable the static listener, set the `--statsd-enabled` flag on the Contour -`bootstrap` command that runs as an init container. +To enable the static listener, set the `--statsd-enabled` flag. +By default, Envoy's stats will be exposed over `0.0.0.0:8002` but can be overridden setting the `--stats-address` and `--stats-port` flags in Contour. ### Configuration Prometheus diff --git a/internal/envoy/config.go b/internal/envoy/config.go index 3bb135168d1..c9d033af1a1 100644 --- a/internal/envoy/config.go +++ b/internal/envoy/config.go @@ -117,7 +117,7 @@ static_resources: protocol: TCP address: 127.0.0.1 port_value: {{ if .AdminPort }}{{ .AdminPort }}{{ else }}9001{{ end }} -{{ if .StatsdEnabled }} listeners: + listeners: - address: socket_address: protocol: TCP @@ -140,9 +140,13 @@ static_resources: route: cluster: service_stats http_filters: + - name: envoy.health_check + config: + endpoint: "/healthz" + pass_through_mode: false - name: envoy.router config: -stats_sinks: +{{ if .StatsdEnabled }}stats_sinks: - name: envoy.statsd config: address: @@ -150,8 +154,7 @@ stats_sinks: protocol: UDP address: {{ if .StatsdAddress }}{{ .StatsdAddress }}{{ else }}127.0.0.1{{ end }} port_value: {{ if .StatsdPort }}{{ .StatsdPort }}{{ else }}9125{{ end }} -{{ end -}} -admin: +{{ end -}}admin: access_log_path: {{ if .AdminAccessLogPath }}{{ .AdminAccessLogPath }}{{ else }}/dev/null{{ end }} address: socket_address: diff --git a/internal/envoy/config_test.go b/internal/envoy/config_test.go index 2f4265d7196..5f1c5948d46 100644 --- a/internal/envoy/config_test.go +++ b/internal/envoy/config_test.go @@ -38,63 +38,6 @@ func TestConfigWriter_WriteYAML(t *testing.T) { grpc_services: - envoy_grpc: cluster_name: contour -static_resources: - clusters: - - name: contour - connect_timeout: { seconds: 5 } - type: STRICT_DNS - hosts: - - socket_address: - address: 127.0.0.1 - port_value: 8001 - lb_policy: ROUND_ROBIN - http2_protocol_options: {} - circuit_breakers: - thresholds: - - priority: high - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 60000000 - max_retries: 50 - - priority: default - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 60000000 - max_retries: 50 - - name: service_stats - connect_timeout: 0.250s - type: LOGICAL_DNS - lb_policy: ROUND_ROBIN - hosts: - - socket_address: - protocol: TCP - address: 127.0.0.1 - port_value: 9001 -admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 9001 -`, - }, - "statsd enabled": { - ConfigWriter: ConfigWriter{ - StatsdEnabled: true, - }, - want: `dynamic_resources: - lds_config: - api_config_source: - api_type: GRPC - grpc_services: - - envoy_grpc: - cluster_name: contour - cds_config: - api_config_source: - api_type: GRPC - grpc_services: - - envoy_grpc: - cluster_name: contour static_resources: clusters: - name: contour @@ -150,16 +93,12 @@ static_resources: route: cluster: service_stats http_filters: + - name: envoy.health_check + config: + endpoint: "/healthz" + pass_through_mode: false - name: envoy.router config: -stats_sinks: - - name: envoy.statsd - config: - address: - socket_address: - protocol: UDP - address: 127.0.0.1 - port_value: 9125 admin: access_log_path: /dev/null address: @@ -242,6 +181,10 @@ static_resources: route: cluster: service_stats http_filters: + - name: envoy.health_check + config: + endpoint: "/healthz" + pass_through_mode: false - name: envoy.router config: stats_sinks: