From d6b515901dcee55f4e82a84891ae3ad2fd47d5ee Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Wed, 22 Jan 2020 15:09:25 -0500 Subject: [PATCH 1/4] Implement envoy shutdown manager Signed-off-by: Steve Sloka --- cmd/contour/contour.go | 6 +- cmd/contour/shutdownmanager.go | 132 ++++++++++++ cmd/contour/shutdownmanager_test.go | 38 ++++ cmd/contour/shutdownmanagercontext.go | 49 +++++ examples/contour/03-envoy.yaml | 36 +++- examples/example-workload/bombardier.job.yaml | 48 +++++ go.mod | 1 + internal/metrics/parser.go | 54 +++++ internal/metrics/parser_test.go | 204 ++++++++++++++++++ 9 files changed, 557 insertions(+), 11 deletions(-) create mode 100644 cmd/contour/shutdownmanager.go create mode 100644 cmd/contour/shutdownmanager_test.go create mode 100644 cmd/contour/shutdownmanagercontext.go create mode 100644 examples/example-workload/bombardier.job.yaml create mode 100644 internal/metrics/parser.go create mode 100644 internal/metrics/parser_test.go diff --git a/cmd/contour/contour.go b/cmd/contour/contour.go index ada8c869404..7782d3c3bcf 100644 --- a/cmd/contour/contour.go +++ b/cmd/contour/contour.go @@ -39,8 +39,10 @@ func main() { log := logrus.StandardLogger() app := kingpin.New("contour", "Contour Kubernetes ingress controller.") - bootstrap, bootstrapCtx := registerBootstrap(app) + envoy := app.Command("envoy", "Sub-command for envoy actions.") + shutdownManager, shutdownManagerCtx := registerShutdownManager(envoy, log) + bootstrap, bootstrapCtx := registerBootstrap(app) certgenApp, certgenConfig := registerCertGen(app) cli := app.Command("cli", "A CLI client for the Contour Kubernetes ingress controller.") @@ -66,6 +68,8 @@ func main() { args := os.Args[1:] switch kingpin.MustParse(app.Parse(args)) { + case shutdownManager.FullCommand(): + check(doShutdownManager(shutdownManagerCtx)) case bootstrap.FullCommand(): doBootstrap(bootstrapCtx) case certgenApp.FullCommand(): diff --git a/cmd/contour/shutdownmanager.go b/cmd/contour/shutdownmanager.go new file mode 100644 index 00000000000..dd7f27ef47e --- /dev/null +++ b/cmd/contour/shutdownmanager.go @@ -0,0 +1,132 @@ +// Copyright © 2020 VMware +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/projectcontour/contour/internal/metrics" + + "github.com/sirupsen/logrus" + + "github.com/projectcontour/contour/internal/workgroup" + "gopkg.in/alecthomas/kingpin.v2" +) + +// handler for /healthz +func (s *shutdownmanagerContext) healthzHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("OK")) + if err != nil { + s.Error(err) + } +} + +// handles /shutdown +func (s *shutdownmanagerContext) shutdownHandler(w http.ResponseWriter, r *http.Request) { + prometheusURL := fmt.Sprintf("http://%s:%d%s", s.envoyHost, s.envoyPort, s.prometheusPath) + envoyAdminURL := fmt.Sprintf("http://%s:%d/healthcheck/fail", s.envoyHost, s.envoyPort) + + // Send shutdown signal to Envoy to start draining connections + err := shutdownEnvoy(envoyAdminURL) + if err != nil { + s.Errorf("Error sending envoy healthcheck fail: %v", err) + } + + s.Infof("Sent healthcheck fail to Envoy...waiting %s before polling for draining connections", s.checkDelay) + time.Sleep(s.checkDelay) + + for { + openConnections, err := getOpenConnections(prometheusURL, s.prometheusStat, s.prometheusValues) + if err != nil { + s.Error(err) + } else { + if openConnections <= s.minOpenConnections { + s.Infof("Found %d open connections with min number of %d connections. Shutting down...", openConnections, s.minOpenConnections) + return + } + s.Infof("Found %d open connections with min number of %d connections. Polling again...", openConnections, s.minOpenConnections) + } + time.Sleep(s.checkInterval) + } +} + +// shutdownEnvoy sends a POST request to /healthcheck/fail to tell Envoy to start draining connections +func shutdownEnvoy(url string) error { + resp, err := http.Post(url, "", nil) + if err != nil { + return fmt.Errorf("creating POST request for URL %q failed: %s", url, err) + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("POST request for URL %q returned HTTP status %s", url, resp.Status) + } + return nil +} + +// getOpenConnections parses a http request to a prometheus endpoint returning the sum of values found +func getOpenConnections(url, prometheusStat string, prometheusValues []string) (int, error) { + // Make request to Envoy Prometheus endpoint + resp, err := http.Get(url) + if err != nil { + return -1, fmt.Errorf("GET request for URL %q failed: %s", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return -1, fmt.Errorf("GET request for URL %q returned HTTP status %s", url, resp.Status) + } + + // Parse Prometheus listener stats for open connections + return metrics.ParseOpenConnections(resp.Body, prometheusStat, prometheusValues) +} + +func doShutdownManager(config *shutdownmanagerContext) error { + var g workgroup.Group + + g.Add(func(stop <-chan struct{}) error { + config.Info("started envoy shutdown manager") + defer config.Info("stopped") + + http.HandleFunc("/healthz", config.healthzHandler) + http.HandleFunc("/shutdown", config.shutdownHandler) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", config.httpServePort), nil)) + + return nil + }) + + return g.Run() +} + +// registerShutdownManager registers the envoy shutdown sub-command and flags +func registerShutdownManager(cmd *kingpin.CmdClause, log logrus.FieldLogger) (*kingpin.CmdClause, *shutdownmanagerContext) { + ctx := &shutdownmanagerContext{ + FieldLogger: log, + } + shutdownmgr := cmd.Command("shutdown-manager", "Start envoy shutdown-manager.") + shutdownmgr.Flag("check-interval", "Time to poll Envoy for open connections.").Default("5s").DurationVar(&ctx.checkInterval) + shutdownmgr.Flag("check-delay", "Time wait before polling Envoy for open connections.").Default("60s").DurationVar(&ctx.checkDelay) + shutdownmgr.Flag("min-open-connections", "Min number of open connections when polling Envoy.").Default("0").IntVar(&ctx.minOpenConnections) + shutdownmgr.Flag("serve-port", "Port to serve the http server on.").Default("8090").IntVar(&ctx.httpServePort) + shutdownmgr.Flag("prometheus-path", "The path to query Envoy's Prometheus HTTP Endpoint.").Default("/stats/prometheus").StringVar(&ctx.prometheusPath) + shutdownmgr.Flag("prometheus-stat", "Prometheus stat to query.").Default("envoy_http_downstream_cx_active").StringVar(&ctx.prometheusStat) + shutdownmgr.Flag("prometheus-values", "Prometheus values to look for in prometheus-stat.").Default("ingress_http", "ingress_https").StringsVar(&ctx.prometheusValues) + shutdownmgr.Flag("envoy-host", "HTTP endpoint for Envoy's stats page.").Default("localhost").StringVar(&ctx.envoyHost) + shutdownmgr.Flag("envoy-port", "HTTP port for Envoy's stats page.").Default("9001").IntVar(&ctx.envoyPort) + + return shutdownmgr, ctx +} diff --git a/cmd/contour/shutdownmanager_test.go b/cmd/contour/shutdownmanager_test.go new file mode 100644 index 00000000000..07c50dd6737 --- /dev/null +++ b/cmd/contour/shutdownmanager_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestShutdownManager_HealthzHandler(t *testing.T) { + // Create a request to pass to our handler + req, err := http.NewRequest("GET", "/healthz", nil) + if err != nil { + t.Fatal(err) + } + + mgr := shutdownmanagerContext{} + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + handler := http.HandlerFunc(mgr.healthzHandler) + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + handler.ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } + + // Check the response body is what we expect. + expected := `OK` + if rr.Body.String() != expected { + t.Errorf("handler returned unexpected body: got %v want %v", + rr.Body.String(), expected) + } +} diff --git a/cmd/contour/shutdownmanagercontext.go b/cmd/contour/shutdownmanagercontext.go new file mode 100644 index 00000000000..bd530dc7307 --- /dev/null +++ b/cmd/contour/shutdownmanagercontext.go @@ -0,0 +1,49 @@ +// Copyright © 2020 VMware +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "time" + + "github.com/sirupsen/logrus" +) + +type shutdownmanagerContext struct { + // checkInterval defines time delay between polling Envoy for open connections + checkInterval time.Duration + + // checkDelay defines time to wait before polling Envoy for open connections + checkDelay time.Duration + + // minOpenConnections defines the minimum amount of connections + // that can be open when polling for active connections in Envoy + minOpenConnections int + + // httpServePort defines the port to serve the http server on + httpServePort int + + // prometheusPath defines the path to query Envoy's Prometheus http Endpoint + prometheusPath string + + // prometheusStat defines the stat to query for in the /stats/prometheus endpoint + prometheusStat string + + // prometheusValues defines the values to query for in the prometheusStat + prometheusValues []string + + envoyHost string + envoyPort int + + logrus.FieldLogger +} diff --git a/examples/contour/03-envoy.yaml b/examples/contour/03-envoy.yaml index ed927a01882..f286eac0e3e 100644 --- a/examples/contour/03-envoy.yaml +++ b/examples/contour/03-envoy.yaml @@ -24,6 +24,26 @@ spec: app: envoy spec: containers: + - command: + - /bin/contour + args: + - envoy + - shutdown-manager + image: stevesloka/contour:dev + imagePullPolicy: Always + lifecycle: + preStop: + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 3 + periodSeconds: 10 + name: shutdown-manager - args: - -c - /config/envoy.json @@ -60,7 +80,7 @@ spec: path: /ready port: 8002 initialDelaySeconds: 3 - periodSeconds: 3 + periodSeconds: 4 volumeMounts: - name: envoy-config mountPath: /config @@ -70,15 +90,10 @@ spec: mountPath: /ca lifecycle: preStop: - exec: - command: - - bash - - -c - - -- - - echo - - -ne - - "POST /healthcheck/fail HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" - - '>/dev/tcp/localhost/9001' + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP initContainers: - args: - bootstrap @@ -108,6 +123,7 @@ spec: fieldRef: fieldPath: metadata.namespace automountServiceAccountToken: false + terminationGracePeriodSeconds: 300 volumes: - name: envoy-config emptyDir: {} diff --git a/examples/example-workload/bombardier.job.yaml b/examples/example-workload/bombardier.job.yaml new file mode 100644 index 00000000000..ae290e3b83a --- /dev/null +++ b/examples/example-workload/bombardier.job.yaml @@ -0,0 +1,48 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + workload: bombardier + generateName: bombardier- +spec: + backoffLimit: 6 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + workload: bombardier + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: workload + operator: In + values: + - bombardier + topologyKey: kubernetes.io/hostname + initContainers: + - command: + - sh + - -c + - sysctl -w net.ipv4.ip_local_port_range="1024 65535" + image: alpine:3.6 + imagePullPolicy: IfNotPresent + name: sysctl-set + securityContext: + privileged: true + containers: + - args: ["-c", "1000", "-d", "300s", "-l", "http://envoy.projectcontour"] + image: alpine/bombardier + imagePullPolicy: Always + name: bombardier + nodeSelector: + workload: bombardier + restartPolicy: OnFailure + + +# 1. Get node labels: kubectl get nodes --show-labels +# 2. Label a node to run the load test workload: kubectl label nodes workload=bombardier +# 3. Label nodes to run the envoy workload: kubectl label nodes workload=envoy diff --git a/go.mod b/go.mod index de992bf6ed5..288d5d88713 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/prometheus/client_golang v1.1.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 + github.com/prometheus/common v0.6.0 github.com/sirupsen/logrus v1.4.2 golang.org/x/tools v0.0.0-20190929041059-e7abfedfabcf // indirect google.golang.org/grpc v1.25.1 diff --git a/internal/metrics/parser.go b/internal/metrics/parser.go new file mode 100644 index 00000000000..f66f14c793f --- /dev/null +++ b/internal/metrics/parser.go @@ -0,0 +1,54 @@ +// Copyright © 2020 VMware +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "io" + + "github.com/prometheus/common/expfmt" +) + +// ParseOpenConnections returns the sum of open connections from a Prometheus HTTP request +func ParseOpenConnections(stats io.Reader, prometheusStat string, prometheusValues []string) (int, error) { + var parser expfmt.TextParser + var openConnections = 0 + + if stats == nil { + return -1, fmt.Errorf("stats input was nil") + } + + // Parse Prometheus http response + metricFamilies, err := parser.TextToMetricFamilies(stats) + if err != nil { + return -1, fmt.Errorf("parsing prometheus text format failed: %v", err) + } + + // Validate stat exists in output + if _, ok := metricFamilies[prometheusStat]; !ok { + return -1, fmt.Errorf("prometheus stat [%s] not found in request result", prometheusStat) + } + + // Look up open connections value + for _, metrics := range metricFamilies[prometheusStat].Metric { + for _, labels := range metrics.Label { + for _, item := range prometheusValues { + if item == *labels.Value { + openConnections += int(*metrics.Gauge.Value) + } + } + } + } + return openConnections, nil +} diff --git a/internal/metrics/parser_test.go b/internal/metrics/parser_test.go new file mode 100644 index 00000000000..b148654875f --- /dev/null +++ b/internal/metrics/parser_test.go @@ -0,0 +1,204 @@ +// Copyright © 2020 VMware +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "io" + "strings" + "testing" + + "github.com/projectcontour/contour/internal/assert" +) + +func TestParseOpenConnections(t *testing.T) { + type testcase struct { + stats io.Reader + prometheusStat string + prometheusValues []string + wantConnections int + wantError error + } + + run := func(t *testing.T, name string, tc testcase) { + t.Helper() + + t.Run(name, func(t *testing.T) { + t.Helper() + + gotConnections, gotError := ParseOpenConnections(tc.stats, tc.prometheusStat, tc.prometheusValues) + assert.Equal(t, tc.wantError, gotError) + assert.Equal(t, tc.wantConnections, gotConnections) + }) + } + + run(t, "nil stats", testcase{ + stats: nil, + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: -1, + wantError: fmt.Errorf("stats input was nil"), + }) + + run(t, "basic http only", testcase{ + stats: strings.NewReader(VALIDHTTP), + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: 4, + wantError: nil, + }) + + run(t, "basic https only", testcase{ + stats: strings.NewReader(VALIDHTTPS), + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: 4, + wantError: nil, + }) + + run(t, "basic both protocols", testcase{ + stats: strings.NewReader(VALIDBOTH), + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: 8, + wantError: nil, + }) + + run(t, "missing values", testcase{ + stats: strings.NewReader(MISSING_STATS), + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: -1, + wantError: fmt.Errorf("prometheus stat [envoy_http_downstream_cx_active] not found in request result"), + }) + + run(t, "invalid stats", testcase{ + stats: strings.NewReader("!!##$$##!!"), + prometheusStat: "envoy_http_downstream_cx_active", + prometheusValues: []string{"ingress_http", "ingress_https"}, + wantConnections: -1, + wantError: fmt.Errorf("parsing prometheus text format failed: text format parsing error in line 1: invalid metric name"), + }) +} + +const ( + VALIDHTTP = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +` + VALIDHTTPS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +` + VALIDBOTH = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_https"} 4 +` + + MISSING_STATS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +` +) From 8106c7c20f0343e4946cfc1fbb192b491946d78b Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Sat, 8 Feb 2020 13:50:16 -0500 Subject: [PATCH 2/4] Add Envoy open connections dashboard to Grafana Signed-off-by: Steve Sloka --- examples/grafana/02-grafana-configmap.yaml | 400 ++++++++++++--------- 1 file changed, 238 insertions(+), 162 deletions(-) diff --git a/examples/grafana/02-grafana-configmap.yaml b/examples/grafana/02-grafana-configmap.yaml index e1171421314..0a33b723c4d 100644 --- a/examples/grafana/02-grafana-configmap.yaml +++ b/examples/grafana/02-grafana-configmap.yaml @@ -1614,39 +1614,10 @@ data: } envoy.json: | { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], "annotations": { "list": [ { + "$$hashKey": "object:348", "builtIn": 1, "datasource": "-- Grafana --", "enable": true, @@ -1660,10 +1631,109 @@ data: "editable": true, "gnetId": null, "graphTooltip": 0, - "id": null, - "iteration": 1533835806222, + "id": 3, + "iteration": 1581187298580, "links": [], "panels": [ + { + "columns": [], + "datasource": null, + "description": "Shows the open connections to the http/https listeners.", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 22, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$$hashKey": "object:658", + "alias": "Time", + "dateFormat": "MM/DD/YY h:mm:ss a", + "pattern": "Time", + "type": "date" + }, + { + "$$hashKey": "object:684", + "alias": "HTTP", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "$$hashKey": "object:710", + "alias": "HTTPS", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "$$hashKey": "object:795", + "alias": "Envoy Pod", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "kubernetes_pod_name", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "$$hashKey": "object:453", + "expr": "sum(envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix=\"ingress_http\"}) by (kubernetes_pod_name)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "$$hashKey": "object:481", + "expr": "sum(envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix=\"ingress_https\"}) by (kubernetes_pod_name)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + } + ], + "title": "Envoy Open Connections", + "transform": "table", + "type": "table" + }, { "aliasColors": {}, "bars": false, @@ -1674,17 +1744,17 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 0, + "x": 12, "y": 0 }, - "id": 3, + "id": 16, "legend": { "alignAsTable": true, "avg": true, "current": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -1705,17 +1775,32 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_http_downstream_rq_total[1m])) by (kubernetes_pod_name)", + "expr": "histogram_quantile(0.9, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_pod_name}}", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}} 90%", "refId": "A" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}} 50% ", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{kubernetes_pod_name}} 99%", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Downstream RPS", + "title": "Downstream Latency", "tooltip": { "shared": true, "sort": 0, @@ -1731,7 +1816,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1758,16 +1843,18 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 6, + "x": 18, "y": 0 }, - "id": 9, + "id": 4, "legend": { "alignAsTable": true, "avg": true, "current": true, + "hideZero": true, "max": true, "min": false, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -1788,9 +1875,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_http_downstream_cx_total[1m])) by (kubernetes_pod_name)", + "expr": "sum(envoy_http_downstream_cx_active) by (kubernetes_pod_name)", "format": "time_series", - "intervalFactor": 1, + "instant": false, + "interval": "", + "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A" } @@ -1798,7 +1887,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Downstream CPS", + "title": "Downstream Total Connections", "tooltip": { "shared": true, "sort": 0, @@ -1841,17 +1930,17 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 0 + "x": 0, + "y": 8 }, - "id": 16, + "id": 3, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideZero": true, "max": true, "min": false, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -1872,32 +1961,17 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", + "expr": "sum(rate(envoy_http_downstream_rq_total[1m])) by (kubernetes_pod_name)", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}} 90%", + "intervalFactor": 2, + "legendFormat": "{{kubernetes_pod_name}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}} 50% ", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(envoy_http_downstream_rq_time_bucket[1m])) by (le, kubernetes_pod_name))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}} 99%", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Downstream Latency", + "title": "Downstream RPS", "tooltip": { "shared": true, "sort": 0, @@ -1913,7 +1987,7 @@ data: }, "yaxes": [ { - "format": "ms", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1940,18 +2014,16 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 18, - "y": 0 + "x": 6, + "y": 8 }, - "id": 4, + "id": 9, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideZero": true, "max": true, "min": false, - "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -1972,11 +2044,9 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(envoy_http_downstream_cx_active) by (kubernetes_pod_name)", + "expr": "sum(rate(envoy_http_downstream_cx_total[1m])) by (kubernetes_pod_name)", "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, + "intervalFactor": 1, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A" } @@ -1984,7 +2054,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Downstream Total Connections", + "title": "Downstream CPS", "tooltip": { "shared": true, "sort": 0, @@ -2023,15 +2093,14 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus", - "description": "Displays the number of Requests per Second being performed against each Upstream.", "fill": 1, "gridPos": { "h": 8, "w": 6, - "x": 0, + "x": 12, "y": 8 }, - "id": 2, + "id": 10, "legend": { "alignAsTable": true, "avg": true, @@ -2039,7 +2108,6 @@ data: "hideZero": true, "max": true, "min": false, - "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -2060,17 +2128,32 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_rq_total{namespace=~\"$Namespace\",service=~\"$Service\"}[1m])) by (service,namespace)", + "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{namespace}}/{{service}}", + "intervalFactor": 1, + "legendFormat": "{{namespace}}/{{service}} 99%", "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}/{{service}} 90%", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{namespace}}/{{service}} 50% ", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream RPS", + "title": "Upstream Latency", "tooltip": { "shared": true, "sort": 0, @@ -2086,7 +2169,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -2113,10 +2196,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 6, + "x": 18, "y": 8 }, - "id": 14, + "id": 15, "legend": { "alignAsTable": true, "avg": true, @@ -2144,7 +2227,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_cx_total{namespace=~\"$Namespace\",service=~\"$Service\"}[1m])) by (namespace, service)", + "expr": "sum(envoy_cluster_upstream_cx_active{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{namespace}}/{{service}}", @@ -2154,7 +2237,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream CPS", + "title": "Upstream Total Connections", "tooltip": { "shared": true, "sort": 0, @@ -2193,14 +2276,15 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus", + "description": "Displays the number of Requests per Second being performed against each Upstream.", "fill": 1, "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 8 + "x": 0, + "y": 16 }, - "id": 10, + "id": 2, "legend": { "alignAsTable": true, "avg": true, @@ -2208,6 +2292,7 @@ data: "hideZero": true, "max": true, "min": false, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -2228,32 +2313,17 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", + "expr": "sum(rate(envoy_cluster_upstream_rq_total{namespace=~\"$Namespace\",service=~\"$Service\"}[1m])) by (service,namespace)", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{namespace}}/{{service}} 99%", + "intervalFactor": 2, + "legendFormat": "{{namespace}}/{{service}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{namespace}}/{{service}} 90%", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{service=~\"$Service\",namespace=~\"$Namespace\"}[1m])) by (le, service, namespace))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{namespace}}/{{service}} 50% ", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream Latency", + "title": "Upstream RPS", "tooltip": { "shared": true, "sort": 0, @@ -2269,7 +2339,7 @@ data: }, "yaxes": [ { - "format": "ms", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2296,10 +2366,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 18, - "y": 8 + "x": 6, + "y": 16 }, - "id": 15, + "id": 14, "legend": { "alignAsTable": true, "avg": true, @@ -2327,7 +2397,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(envoy_cluster_upstream_cx_active{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", + "expr": "sum(rate(envoy_cluster_upstream_cx_total{namespace=~\"$Namespace\",service=~\"$Service\"}[1m])) by (namespace, service)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{namespace}}/{{service}}", @@ -2337,7 +2407,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream Total Connections", + "title": "Upstream CPS", "tooltip": { "shared": true, "sort": 0, @@ -2380,15 +2450,14 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 0, + "x": 12, "y": 16 }, - "id": 5, + "id": 12, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideEmpty": false, "hideZero": true, "max": true, "min": false, @@ -2413,7 +2482,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=~\"2\"}[1m])) by (namespace,service)", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"4\"}[1m])) by (namespace,service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2423,7 +2492,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream 2xx Responses", + "title": "Upstream 4xx Responses", "tooltip": { "shared": true, "sort": 0, @@ -2466,10 +2535,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 6, + "x": 18, "y": 16 }, - "id": 11, + "id": 13, "legend": { "alignAsTable": true, "avg": true, @@ -2498,7 +2567,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"3\"}[1m])) by (namespace,service)", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"5\"}[1m])) by (namespace,service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2508,7 +2577,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream 3xx Responses", + "title": "Upstream 5xx Responses", "tooltip": { "shared": true, "sort": 0, @@ -2551,14 +2620,15 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 16 + "x": 0, + "y": 24 }, - "id": 12, + "id": 5, "legend": { "alignAsTable": true, "avg": true, "current": true, + "hideEmpty": false, "hideZero": true, "max": true, "min": false, @@ -2583,7 +2653,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"4\"}[1m])) by (namespace,service)", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=~\"2\"}[1m])) by (namespace,service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2593,7 +2663,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream 4xx Responses", + "title": "Upstream 2xx Responses", "tooltip": { "shared": true, "sort": 0, @@ -2636,10 +2706,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 18, - "y": 16 + "x": 6, + "y": 24 }, - "id": 13, + "id": 11, "legend": { "alignAsTable": true, "avg": true, @@ -2668,7 +2738,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"5\"}[1m])) by (namespace,service)", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{namespace=~\"$Namespace\",service=~\"$Service\",envoy_response_code_class=\"3\"}[1m])) by (namespace,service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2678,7 +2748,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Upstream 5xx Responses", + "title": "Upstream 3xx Responses", "tooltip": { "shared": true, "sort": 0, @@ -2722,10 +2792,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 0, + "x": 12, "y": 24 }, - "id": 17, + "id": 18, "legend": { "alignAsTable": true, "avg": true, @@ -2754,7 +2824,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service) / avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", + "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2764,7 +2834,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Endpoint Percentage Health", + "title": "Healthy Endpoints", "tooltip": { "shared": true, "sort": 0, @@ -2780,7 +2850,7 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2808,10 +2878,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 6, + "x": 18, "y": 24 }, - "id": 19, + "id": 20, "legend": { "alignAsTable": true, "avg": true, @@ -2840,7 +2910,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", + "expr": "avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service) - avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2850,7 +2920,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total Endpoints", + "title": "Unhealthy Endpoints", "tooltip": { "shared": true, "sort": 0, @@ -2894,10 +2964,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 24 + "x": 0, + "y": 32 }, - "id": 18, + "id": 17, "legend": { "alignAsTable": true, "avg": true, @@ -2926,7 +2996,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", + "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service) / avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -2936,7 +3006,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Healthy Endpoints", + "title": "Endpoint Percentage Health", "tooltip": { "shared": true, "sort": 0, @@ -2952,7 +3022,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -2980,10 +3050,10 @@ data: "gridPos": { "h": 8, "w": 6, - "x": 18, - "y": 24 + "x": 6, + "y": 32 }, - "id": 20, + "id": 19, "legend": { "alignAsTable": true, "avg": true, @@ -3012,7 +3082,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service) - avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", + "expr": "avg(envoy_cluster_membership_total{namespace=~\"$Namespace\",service=~\"$Service\"}) by (namespace, service)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}/{{service}}", @@ -3022,7 +3092,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Unhealthy Endpoints", + "title": "Total Endpoints", "tooltip": { "shared": true, "sort": 0, @@ -3064,7 +3134,10 @@ data: "list": [ { "allValue": ".*", - "current": {}, + "current": { + "text": "All", + "value": "$__all" + }, "datasource": "prometheus", "hide": 0, "includeAll": true, @@ -3084,7 +3157,10 @@ data: }, { "allValue": ".*", - "current": {}, + "current": { + "text": "All", + "value": "$__all" + }, "datasource": "prometheus", "hide": 0, "includeAll": true, @@ -3136,7 +3212,7 @@ data: "timezone": "", "title": "Envoy Metrics", "uid": "khVnG8iiz", - "version": 2 + "version": 4 } --- From ab8a1a8f8a5d2530f8f3f70fa1cb8650810c4bf5 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Thu, 13 Feb 2020 15:29:43 -0500 Subject: [PATCH 3/4] Update rendered quickstart example utilizing the shutdown-manager Signed-off-by: Steve Sloka --- examples/contour/03-envoy.yaml | 2 +- examples/render/contour.yaml | 36 ++++++++++++++++++++-------- site/docs/master/shutdown-manager.md | 0 3 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 site/docs/master/shutdown-manager.md diff --git a/examples/contour/03-envoy.yaml b/examples/contour/03-envoy.yaml index f286eac0e3e..32c85f487d4 100644 --- a/examples/contour/03-envoy.yaml +++ b/examples/contour/03-envoy.yaml @@ -29,7 +29,7 @@ spec: args: - envoy - shutdown-manager - image: stevesloka/contour:dev + image: docker.io/projectcontour/contour:master imagePullPolicy: Always lifecycle: preStop: diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 98940c0de2c..217376f4d9f 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -1612,6 +1612,26 @@ spec: app: envoy spec: containers: + - command: + - /bin/contour + args: + - envoy + - shutdown-manager + image: docker.io/projectcontour/contour:master + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 3 + periodSeconds: 10 + name: shutdown-manager - args: - -c - /config/envoy.json @@ -1648,7 +1668,7 @@ spec: path: /ready port: 8002 initialDelaySeconds: 3 - periodSeconds: 3 + periodSeconds: 4 volumeMounts: - name: envoy-config mountPath: /config @@ -1658,15 +1678,10 @@ spec: mountPath: /ca lifecycle: preStop: - exec: - command: - - bash - - -c - - -- - - echo - - -ne - - "POST /healthcheck/fail HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" - - '>/dev/tcp/localhost/9001' + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP initContainers: - args: - bootstrap @@ -1696,6 +1711,7 @@ spec: fieldRef: fieldPath: metadata.namespace automountServiceAccountToken: false + terminationGracePeriodSeconds: 300 volumes: - name: envoy-config emptyDir: {} diff --git a/site/docs/master/shutdown-manager.md b/site/docs/master/shutdown-manager.md new file mode 100644 index 00000000000..e69de29bb2d From 572fdc2e6476616a66efca8a8874cf4bd981d35c Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Thu, 13 Feb 2020 16:17:39 -0500 Subject: [PATCH 4/4] Add docs for Envoy shutdown manager Signed-off-by: Steve Sloka --- cmd/contour/contour.go | 2 +- cmd/contour/shutdownmanager.go | 150 +++++++++---- cmd/contour/shutdownmanager_test.go | 172 +++++++++++++++ cmd/contour/shutdownmanagercontext.go | 49 ----- examples/example-workload/bombardier.job.yaml | 48 ----- internal/metrics/parser.go | 54 ----- internal/metrics/parser_test.go | 204 ------------------ site/_data/master-toc.yml | 2 + site/_scss/site/common/_core.scss | 6 + site/docs/master/deploy-options.md | 8 + site/docs/master/img/shutdownmanager.png | Bin 0 -> 40847 bytes site/docs/master/redeploy-envoy.md | 65 ++++++ site/docs/master/shutdown-manager.md | 0 13 files changed, 359 insertions(+), 401 deletions(-) delete mode 100644 cmd/contour/shutdownmanagercontext.go delete mode 100644 examples/example-workload/bombardier.job.yaml delete mode 100644 internal/metrics/parser.go delete mode 100644 internal/metrics/parser_test.go create mode 100644 site/docs/master/img/shutdownmanager.png create mode 100644 site/docs/master/redeploy-envoy.md delete mode 100644 site/docs/master/shutdown-manager.md diff --git a/cmd/contour/contour.go b/cmd/contour/contour.go index 7782d3c3bcf..686d5b30999 100644 --- a/cmd/contour/contour.go +++ b/cmd/contour/contour.go @@ -69,7 +69,7 @@ func main() { args := os.Args[1:] switch kingpin.MustParse(app.Parse(args)) { case shutdownManager.FullCommand(): - check(doShutdownManager(shutdownManagerCtx)) + doShutdownManager(shutdownManagerCtx) case bootstrap.FullCommand(): doBootstrap(bootstrapCtx) case certgenApp.FullCommand(): diff --git a/cmd/contour/shutdownmanager.go b/cmd/contour/shutdownmanager.go index dd7f27ef47e..af8269470df 100644 --- a/cmd/contour/shutdownmanager.go +++ b/cmd/contour/shutdownmanager.go @@ -15,51 +15,92 @@ package main import ( "fmt" + "io" "log" "net/http" "time" - "github.com/projectcontour/contour/internal/metrics" + "github.com/projectcontour/contour/internal/contour" + + "github.com/prometheus/common/expfmt" "github.com/sirupsen/logrus" - "github.com/projectcontour/contour/internal/workgroup" "gopkg.in/alecthomas/kingpin.v2" ) -// handler for /healthz +const ( + prometheusURL = "http://localhost:9001/stats/prometheus" + healthcheckFailURL = "http://localhost:9001/healthcheck/fail" + prometheusStat = "envoy_http_downstream_cx_active" +) + +func prometheusLabels() []string { + return []string{contour.ENVOY_HTTP_LISTENER, contour.ENVOY_HTTPS_LISTENER} +} + +type shutdownmanagerContext struct { + // checkInterval defines time delay between polling Envoy for open connections + checkInterval time.Duration + + // checkDelay defines time to wait before polling Envoy for open connections + checkDelay time.Duration + + // minOpenConnections defines the minimum amount of connections + // that can be open when polling for active connections in Envoy + minOpenConnections int + + // httpServePort defines what port the shutdown-manager listens on + httpServePort int + + logrus.FieldLogger +} + +func newShutdownManagerContext() *shutdownmanagerContext { + // Set defaults for parameters which are then overridden via flags, ENV, or ConfigFile + return &shutdownmanagerContext{ + checkInterval: 5 * time.Second, + checkDelay: 60 * time.Second, + minOpenConnections: 0, + httpServePort: 8090, + } +} + +// handles the /healthz endpoint which is used for the shutdown-manager's liveness probe func (s *shutdownmanagerContext) healthzHandler(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte("OK")) - if err != nil { + http.StatusText(http.StatusOK) + if _, err := w.Write([]byte("OK")); err != nil { s.Error(err) } } -// handles /shutdown +// shutdownHandler handles the /shutdown endpoint which should be called from a pod preStop hook, +// where it will block pod shutdown until envoy is able to drain connections to below the min-open threshold func (s *shutdownmanagerContext) shutdownHandler(w http.ResponseWriter, r *http.Request) { - prometheusURL := fmt.Sprintf("http://%s:%d%s", s.envoyHost, s.envoyPort, s.prometheusPath) - envoyAdminURL := fmt.Sprintf("http://%s:%d/healthcheck/fail", s.envoyHost, s.envoyPort) - // Send shutdown signal to Envoy to start draining connections - err := shutdownEnvoy(envoyAdminURL) + s.Infof("failing envoy healthchecks") + err := shutdownEnvoy(healthcheckFailURL) if err != nil { - s.Errorf("Error sending envoy healthcheck fail: %v", err) + s.Errorf("error sending envoy healthcheck fail: %v", err) } - s.Infof("Sent healthcheck fail to Envoy...waiting %s before polling for draining connections", s.checkDelay) + s.Infof("waiting %s before polling for draining connections", s.checkDelay) time.Sleep(s.checkDelay) for { - openConnections, err := getOpenConnections(prometheusURL, s.prometheusStat, s.prometheusValues) + openConnections, err := getOpenConnections(prometheusURL) if err != nil { s.Error(err) } else { if openConnections <= s.minOpenConnections { - s.Infof("Found %d open connections with min number of %d connections. Shutting down...", openConnections, s.minOpenConnections) + s.WithField("open_connections", openConnections). + WithField("min_connections", s.minOpenConnections). + Info("min number of open connections found, shutting down") return } - s.Infof("Found %d open connections with min number of %d connections. Polling again...", openConnections, s.minOpenConnections) + s.WithField("open_connections", openConnections). + WithField("min_connections", s.minOpenConnections). + Info("polled open connections") } time.Sleep(s.checkInterval) } @@ -69,64 +110,83 @@ func (s *shutdownmanagerContext) shutdownHandler(w http.ResponseWriter, r *http. func shutdownEnvoy(url string) error { resp, err := http.Post(url, "", nil) if err != nil { - return fmt.Errorf("creating POST request for URL %q failed: %s", url, err) + return fmt.Errorf("creating healthcheck fail post request failed: %s", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("POST request for URL %q returned HTTP status %s", url, resp.Status) + return fmt.Errorf("post request for url %q returned http status %s", url, resp.Status) } return nil } // getOpenConnections parses a http request to a prometheus endpoint returning the sum of values found -func getOpenConnections(url, prometheusStat string, prometheusValues []string) (int, error) { +func getOpenConnections(url string) (int, error) { // Make request to Envoy Prometheus endpoint resp, err := http.Get(url) if err != nil { - return -1, fmt.Errorf("GET request for URL %q failed: %s", url, err) + return -1, fmt.Errorf("get request for metrics failed: %s", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return -1, fmt.Errorf("GET request for URL %q returned HTTP status %s", url, resp.Status) + return -1, fmt.Errorf("get request for metrics failed with http status %s", resp.Status) } // Parse Prometheus listener stats for open connections - return metrics.ParseOpenConnections(resp.Body, prometheusStat, prometheusValues) + return parseOpenConnections(resp.Body) } -func doShutdownManager(config *shutdownmanagerContext) error { - var g workgroup.Group +// parseOpenConnections returns the sum of open connections from a Prometheus HTTP request +func parseOpenConnections(stats io.Reader) (int, error) { + var parser expfmt.TextParser + openConnections := 0 + + if stats == nil { + return -1, fmt.Errorf("stats input was nil") + } - g.Add(func(stop <-chan struct{}) error { - config.Info("started envoy shutdown manager") - defer config.Info("stopped") + // Parse Prometheus http response + metricFamilies, err := parser.TextToMetricFamilies(stats) + if err != nil { + return -1, fmt.Errorf("parsing Prometheus text format failed: %v", err) + } + + // Validate stat exists in output + if _, ok := metricFamilies[prometheusStat]; !ok { + return -1, fmt.Errorf("error finding Prometheus stat %q in the request result", prometheusStat) + } - http.HandleFunc("/healthz", config.healthzHandler) - http.HandleFunc("/shutdown", config.shutdownHandler) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", config.httpServePort), nil)) + // Look up open connections value + for _, metrics := range metricFamilies[prometheusStat].Metric { + for _, labels := range metrics.Label { + for _, item := range prometheusLabels() { + if item == labels.GetValue() { + openConnections += int(metrics.Gauge.GetValue()) + } + } + } + } + return openConnections, nil +} - return nil - }) +func doShutdownManager(config *shutdownmanagerContext) { + config.Info("started envoy shutdown manager") + defer config.Info("stopped") - return g.Run() + http.HandleFunc("/healthz", config.healthzHandler) + http.HandleFunc("/shutdown", config.shutdownHandler) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", config.httpServePort), nil)) } // registerShutdownManager registers the envoy shutdown sub-command and flags func registerShutdownManager(cmd *kingpin.CmdClause, log logrus.FieldLogger) (*kingpin.CmdClause, *shutdownmanagerContext) { - ctx := &shutdownmanagerContext{ - FieldLogger: log, - } + ctx := newShutdownManagerContext() + ctx.FieldLogger = log.WithField("context", "shutdown-manager") + shutdownmgr := cmd.Command("shutdown-manager", "Start envoy shutdown-manager.") - shutdownmgr.Flag("check-interval", "Time to poll Envoy for open connections.").Default("5s").DurationVar(&ctx.checkInterval) + shutdownmgr.Flag("check-interval", "Time to poll Envoy for open connections.").DurationVar(&ctx.checkInterval) shutdownmgr.Flag("check-delay", "Time wait before polling Envoy for open connections.").Default("60s").DurationVar(&ctx.checkDelay) - shutdownmgr.Flag("min-open-connections", "Min number of open connections when polling Envoy.").Default("0").IntVar(&ctx.minOpenConnections) - shutdownmgr.Flag("serve-port", "Port to serve the http server on.").Default("8090").IntVar(&ctx.httpServePort) - shutdownmgr.Flag("prometheus-path", "The path to query Envoy's Prometheus HTTP Endpoint.").Default("/stats/prometheus").StringVar(&ctx.prometheusPath) - shutdownmgr.Flag("prometheus-stat", "Prometheus stat to query.").Default("envoy_http_downstream_cx_active").StringVar(&ctx.prometheusStat) - shutdownmgr.Flag("prometheus-values", "Prometheus values to look for in prometheus-stat.").Default("ingress_http", "ingress_https").StringsVar(&ctx.prometheusValues) - shutdownmgr.Flag("envoy-host", "HTTP endpoint for Envoy's stats page.").Default("localhost").StringVar(&ctx.envoyHost) - shutdownmgr.Flag("envoy-port", "HTTP port for Envoy's stats page.").Default("9001").IntVar(&ctx.envoyPort) - + shutdownmgr.Flag("min-open-connections", "Min number of open connections when polling Envoy.").IntVar(&ctx.minOpenConnections) + shutdownmgr.Flag("serve-port", "Port to serve the http server on.").IntVar(&ctx.httpServePort) return shutdownmgr, ctx } diff --git a/cmd/contour/shutdownmanager_test.go b/cmd/contour/shutdownmanager_test.go index 07c50dd6737..fda913f8ff3 100644 --- a/cmd/contour/shutdownmanager_test.go +++ b/cmd/contour/shutdownmanager_test.go @@ -1,9 +1,14 @@ package main import ( + "fmt" + "io" "net/http" "net/http/httptest" + "strings" "testing" + + "github.com/projectcontour/contour/internal/assert" ) func TestShutdownManager_HealthzHandler(t *testing.T) { @@ -36,3 +41,170 @@ func TestShutdownManager_HealthzHandler(t *testing.T) { rr.Body.String(), expected) } } + +func TestParseOpenConnections(t *testing.T) { + type testcase struct { + stats io.Reader + wantConnections int + wantError error + } + + run := func(t *testing.T, name string, tc testcase) { + t.Helper() + + t.Run(name, func(t *testing.T) { + t.Helper() + + gotConnections, gotError := parseOpenConnections(tc.stats) + assert.Equal(t, tc.wantError, gotError) + assert.Equal(t, tc.wantConnections, gotConnections) + }) + } + + run(t, "nil stats", testcase{ + stats: nil, + wantConnections: -1, + wantError: fmt.Errorf("stats input was nil"), + }) + + run(t, "basic http only", testcase{ + stats: strings.NewReader(VALIDHTTP), + wantConnections: 4, + wantError: nil, + }) + + run(t, "basic https only", testcase{ + stats: strings.NewReader(VALIDHTTPS), + wantConnections: 4, + wantError: nil, + }) + + run(t, "basic both protocols", testcase{ + stats: strings.NewReader(VALIDBOTH), + wantConnections: 8, + wantError: nil, + }) + + run(t, "missing values", testcase{ + stats: strings.NewReader(MISSING_STATS), + wantConnections: -1, + wantError: fmt.Errorf("prometheus stat [envoy_http_downstream_cx_active] not found in request result"), + }) + + run(t, "invalid stats", testcase{ + stats: strings.NewReader("!!##$$##!!"), + wantConnections: -1, + wantError: fmt.Errorf("parsing prometheus text format failed: text format parsing error in line 1: invalid metric name"), + }) +} + +const ( + VALIDHTTP = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +` + VALIDHTTPS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +` + VALIDBOTH = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_https"} 4 +` + + MISSING_STATS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 +# TYPE envoy_http_downstream_cx_ssl_active gauge +envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 +# TYPE envoy_server_total_connections gauge +envoy_server_total_connections{} 1 +# TYPE envoy_runtime_num_layers gauge +envoy_runtime_num_layers{} 2 +# TYPE envoy_server_parent_connections gauge +envoy_server_parent_connections{} 0 +# TYPE envoy_server_stats_recent_lookups gauge +envoy_server_stats_recent_lookups{} 0 +# TYPE envoy_cluster_manager_warming_clusters gauge +envoy_cluster_manager_warming_clusters{} 0 +# TYPE envoy_server_days_until_first_cert_expiring gauge +envoy_server_days_until_first_cert_expiring{} 82 +# TYPE envoy_server_hot_restart_epoch gauge +envoy_server_hot_restart_epoch{} 0 +` +) diff --git a/cmd/contour/shutdownmanagercontext.go b/cmd/contour/shutdownmanagercontext.go deleted file mode 100644 index bd530dc7307..00000000000 --- a/cmd/contour/shutdownmanagercontext.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2020 VMware -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "time" - - "github.com/sirupsen/logrus" -) - -type shutdownmanagerContext struct { - // checkInterval defines time delay between polling Envoy for open connections - checkInterval time.Duration - - // checkDelay defines time to wait before polling Envoy for open connections - checkDelay time.Duration - - // minOpenConnections defines the minimum amount of connections - // that can be open when polling for active connections in Envoy - minOpenConnections int - - // httpServePort defines the port to serve the http server on - httpServePort int - - // prometheusPath defines the path to query Envoy's Prometheus http Endpoint - prometheusPath string - - // prometheusStat defines the stat to query for in the /stats/prometheus endpoint - prometheusStat string - - // prometheusValues defines the values to query for in the prometheusStat - prometheusValues []string - - envoyHost string - envoyPort int - - logrus.FieldLogger -} diff --git a/examples/example-workload/bombardier.job.yaml b/examples/example-workload/bombardier.job.yaml deleted file mode 100644 index ae290e3b83a..00000000000 --- a/examples/example-workload/bombardier.job.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - labels: - workload: bombardier - generateName: bombardier- -spec: - backoffLimit: 6 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - workload: bombardier - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: workload - operator: In - values: - - bombardier - topologyKey: kubernetes.io/hostname - initContainers: - - command: - - sh - - -c - - sysctl -w net.ipv4.ip_local_port_range="1024 65535" - image: alpine:3.6 - imagePullPolicy: IfNotPresent - name: sysctl-set - securityContext: - privileged: true - containers: - - args: ["-c", "1000", "-d", "300s", "-l", "http://envoy.projectcontour"] - image: alpine/bombardier - imagePullPolicy: Always - name: bombardier - nodeSelector: - workload: bombardier - restartPolicy: OnFailure - - -# 1. Get node labels: kubectl get nodes --show-labels -# 2. Label a node to run the load test workload: kubectl label nodes workload=bombardier -# 3. Label nodes to run the envoy workload: kubectl label nodes workload=envoy diff --git a/internal/metrics/parser.go b/internal/metrics/parser.go deleted file mode 100644 index f66f14c793f..00000000000 --- a/internal/metrics/parser.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2020 VMware -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metrics - -import ( - "fmt" - "io" - - "github.com/prometheus/common/expfmt" -) - -// ParseOpenConnections returns the sum of open connections from a Prometheus HTTP request -func ParseOpenConnections(stats io.Reader, prometheusStat string, prometheusValues []string) (int, error) { - var parser expfmt.TextParser - var openConnections = 0 - - if stats == nil { - return -1, fmt.Errorf("stats input was nil") - } - - // Parse Prometheus http response - metricFamilies, err := parser.TextToMetricFamilies(stats) - if err != nil { - return -1, fmt.Errorf("parsing prometheus text format failed: %v", err) - } - - // Validate stat exists in output - if _, ok := metricFamilies[prometheusStat]; !ok { - return -1, fmt.Errorf("prometheus stat [%s] not found in request result", prometheusStat) - } - - // Look up open connections value - for _, metrics := range metricFamilies[prometheusStat].Metric { - for _, labels := range metrics.Label { - for _, item := range prometheusValues { - if item == *labels.Value { - openConnections += int(*metrics.Gauge.Value) - } - } - } - } - return openConnections, nil -} diff --git a/internal/metrics/parser_test.go b/internal/metrics/parser_test.go deleted file mode 100644 index b148654875f..00000000000 --- a/internal/metrics/parser_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright © 2020 VMware -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metrics - -import ( - "fmt" - "io" - "strings" - "testing" - - "github.com/projectcontour/contour/internal/assert" -) - -func TestParseOpenConnections(t *testing.T) { - type testcase struct { - stats io.Reader - prometheusStat string - prometheusValues []string - wantConnections int - wantError error - } - - run := func(t *testing.T, name string, tc testcase) { - t.Helper() - - t.Run(name, func(t *testing.T) { - t.Helper() - - gotConnections, gotError := ParseOpenConnections(tc.stats, tc.prometheusStat, tc.prometheusValues) - assert.Equal(t, tc.wantError, gotError) - assert.Equal(t, tc.wantConnections, gotConnections) - }) - } - - run(t, "nil stats", testcase{ - stats: nil, - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: -1, - wantError: fmt.Errorf("stats input was nil"), - }) - - run(t, "basic http only", testcase{ - stats: strings.NewReader(VALIDHTTP), - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: 4, - wantError: nil, - }) - - run(t, "basic https only", testcase{ - stats: strings.NewReader(VALIDHTTPS), - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: 4, - wantError: nil, - }) - - run(t, "basic both protocols", testcase{ - stats: strings.NewReader(VALIDBOTH), - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: 8, - wantError: nil, - }) - - run(t, "missing values", testcase{ - stats: strings.NewReader(MISSING_STATS), - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: -1, - wantError: fmt.Errorf("prometheus stat [envoy_http_downstream_cx_active] not found in request result"), - }) - - run(t, "invalid stats", testcase{ - stats: strings.NewReader("!!##$$##!!"), - prometheusStat: "envoy_http_downstream_cx_active", - prometheusValues: []string{"ingress_http", "ingress_https"}, - wantConnections: -1, - wantError: fmt.Errorf("parsing prometheus text format failed: text format parsing error in line 1: invalid metric name"), - }) -} - -const ( - VALIDHTTP = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -# TYPE envoy_http_downstream_cx_ssl_active gauge -envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 -# TYPE envoy_server_total_connections gauge -envoy_server_total_connections{} 1 -# TYPE envoy_runtime_num_layers gauge -envoy_runtime_num_layers{} 2 -# TYPE envoy_server_parent_connections gauge -envoy_server_parent_connections{} 0 -# TYPE envoy_server_stats_recent_lookups gauge -envoy_server_stats_recent_lookups{} 0 -# TYPE envoy_cluster_manager_warming_clusters gauge -envoy_cluster_manager_warming_clusters{} 0 -# TYPE envoy_server_days_until_first_cert_expiring gauge -envoy_server_days_until_first_cert_expiring{} 82 -# TYPE envoy_server_hot_restart_epoch gauge -envoy_server_hot_restart_epoch{} 0 -# TYPE envoy_http_downstream_cx_active gauge -envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 -` - VALIDHTTPS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -# TYPE envoy_http_downstream_cx_ssl_active gauge -envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 -# TYPE envoy_server_total_connections gauge -envoy_server_total_connections{} 1 -# TYPE envoy_runtime_num_layers gauge -envoy_runtime_num_layers{} 2 -# TYPE envoy_server_parent_connections gauge -envoy_server_parent_connections{} 0 -# TYPE envoy_server_stats_recent_lookups gauge -envoy_server_stats_recent_lookups{} 0 -# TYPE envoy_cluster_manager_warming_clusters gauge -envoy_cluster_manager_warming_clusters{} 0 -# TYPE envoy_server_days_until_first_cert_expiring gauge -envoy_server_days_until_first_cert_expiring{} 82 -# TYPE envoy_server_hot_restart_epoch gauge -envoy_server_hot_restart_epoch{} 0 -# TYPE envoy_http_downstream_cx_active gauge -envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 -` - VALIDBOTH = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -# TYPE envoy_http_downstream_cx_ssl_active gauge -envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 -# TYPE envoy_server_total_connections gauge -envoy_server_total_connections{} 1 -# TYPE envoy_runtime_num_layers gauge -envoy_runtime_num_layers{} 2 -# TYPE envoy_server_parent_connections gauge -envoy_server_parent_connections{} 0 -# TYPE envoy_server_stats_recent_lookups gauge -envoy_server_stats_recent_lookups{} 0 -# TYPE envoy_cluster_manager_warming_clusters gauge -envoy_cluster_manager_warming_clusters{} 0 -# TYPE envoy_server_days_until_first_cert_expiring gauge -envoy_server_days_until_first_cert_expiring{} 82 -# TYPE envoy_server_hot_restart_epoch gauge -envoy_server_hot_restart_epoch{} 0 -# TYPE envoy_http_downstream_cx_active gauge -envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4 -envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_https"} 4 -` - - MISSING_STATS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_version{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_service-stats_9001"} 0 -# TYPE envoy_http_downstream_cx_ssl_active gauge -envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 -# TYPE envoy_server_total_connections gauge -envoy_server_total_connections{} 1 -# TYPE envoy_runtime_num_layers gauge -envoy_runtime_num_layers{} 2 -# TYPE envoy_server_parent_connections gauge -envoy_server_parent_connections{} 0 -# TYPE envoy_server_stats_recent_lookups gauge -envoy_server_stats_recent_lookups{} 0 -# TYPE envoy_cluster_manager_warming_clusters gauge -envoy_cluster_manager_warming_clusters{} 0 -# TYPE envoy_server_days_until_first_cert_expiring gauge -envoy_server_days_until_first_cert_expiring{} 82 -# TYPE envoy_server_hot_restart_epoch gauge -envoy_server_hot_restart_epoch{} 0 -` -) diff --git a/site/_data/master-toc.yml b/site/_data/master-toc.yml index 06db5e00493..39c350be328 100644 --- a/site/_data/master-toc.yml +++ b/site/_data/master-toc.yml @@ -23,6 +23,8 @@ toc: link: /resources/upgrading - page: Enabling TLS between Envoy and Contour url: /grpc-tls-howto + - page: Redeploy Envoy + url: /redeploy-envoy - title: Guides subfolderitems: - page: Cert-Manager diff --git a/site/_scss/site/common/_core.scss b/site/_scss/site/common/_core.scss index c7c04d018b9..62e0a786cd4 100644 --- a/site/_scss/site/common/_core.scss +++ b/site/_scss/site/common/_core.scss @@ -10,4 +10,10 @@ a { &.light { color: $white !important; } +} + +.center-image +{ + margin: 0 auto; + display: block; } \ No newline at end of file diff --git a/site/docs/master/deploy-options.md b/site/docs/master/deploy-options.md index 353ad1a4b73..6c1e0b169a3 100644 --- a/site/docs/master/deploy-options.md +++ b/site/docs/master/deploy-options.md @@ -208,6 +208,13 @@ Envoy will listen directly on port 8080 on each host that it is running. This is best paired with a DaemonSet (perhaps paired with Node affinity) to ensure that a single instance of Contour runs on each Node. See the [AWS NLB tutorial][10] as an example. +### Upgrading Contour/Envoy + +At times it's needed to upgrade Contour, the version of Envoy, or both. +The included `shutdown-manager` can assist with watching Envoy for open connections while draining and give signal back to Kubernetes as to when it's fine to delete Envoy pods during this process. + +See the [redeploy envoy][11] docs for more information. + ## Running Contour in tandem with another ingress controller If you're running multiple ingress controllers, or running on a cloudprovider that natively handles ingress, @@ -233,3 +240,4 @@ $ kubectl delete ns projectcontour [8]: {% link getting-started.md %} [9]: httpproxy.md [10]: {% link _guides/deploy-aws-nlb.md %} +[11]: redeploy-envoy.md diff --git a/site/docs/master/img/shutdownmanager.png b/site/docs/master/img/shutdownmanager.png new file mode 100644 index 0000000000000000000000000000000000000000..eadeb197886f2a3c29d37b8821573f4d3c0710cc GIT binary patch literal 40847 zcmeFZcT`hv_b*5XrGxa|RC*xvCM5~Ig%WxTp?3l#6s0Kw3erJE5Rqb`ND&1oBBFp4 zK|vG+1OWx<2!e8-1K)S<``($kcmA5WYt3&hl#r7==c#*t_Gf?gNwqN3r$5Pkl7xhW z-p~MnCLtl?1OJ?8sK7hUp6oItBqtc~NIQHW!OJ%QM$=Gz=Wx-TQb5fx!k?UrR5nuY-f9U9_&Pyp5O6an~bpVd1{PLB}7H zlb2DJkvqPCCx+mT-}Md-_6`8|lox{|!328xx_gJY2ZHv4+(B24X3zr6fb>6K^!;br zAttYDsD!dmkh4G; z1vuFGhkG~#;)7xO@BoZQAk4}#FdP%+>!%Oq*BE>hjFh0@ZyoMq>>ccG0=Mx+%2|ZD zTSR;LgNs2b7#{~!1-zM}ih^DwJTlr819On~iwFty!l_0P5N1(CKci^yLH7tb8!HcM zf~k?NLQq(U30@Zo8umAVDHs?>1R6&myh9PXR;D&K271AGT^~z(9Zwqv9iM1?Kp583 zR}Y3W4mI>Lib8uK{0&i|w&1(IRuL*jwl;FgXitK?r>O->1#jVDWJ@4KDVh)zgF<0? zc)4f`Q?wV+&Qnhn9qJ!p<_C}V569!MM1$zyKsy}P)B&ktV(keJM27i=1Ui@~`olbM zVP>jW0@2WtXm6@0?`!U9igPztafq_V%F9Li8%6lxe2nFBhCUb*bG&>g+&;v|UC+S}YoiI4Y zHr&_H(->u<7iO!E(YKDYjPP|g*SE2@Ht;ku(e(_+2K%do8<iOdk;YK)fogkA4 z9Sa*}w0{7?NEwT>wUmoO1_bK|7!n=e`d-*D%ZP9t#V9md!Q0LftD}dO*Kx4*)kTN8 zt3;vT5on}dn2|-Gp@FKozMgletwNZhJ5tGxXkqJRDn~Fw!+jOyy_KvCZ4HAgRU&Pa z;eL9EsNg^!f?=SSwWXq;k9Dw3xVfT{9@^5>I!s>C9Oq@L>Wjcx=s6G+OpvDVXop~2 zKtQ;sRYY`DxPBBEu#Ex^;f}^c%jpLa!(gZ|gbEtWt5u|-MWlb24%Xhn(jwSX7pJ2S z=E~DX&J$@LrJ7873y{%30-l}kS zOQc?)r!U?#!WgTE)yKdLkqA{e_dw48BmF=x3oE?{D?+$FQpryl6J)Nay(G4GrL?QO5eAhVq8N$`~Zv*hmqftZM7)Zs%#~hqS;&g_?Wo8kr&G&0s`N)o>5I zYNV|dF~lq)LO;MOz%nQ#I>;Vt4Tj=i85p7%0nQQziFQ|22@L{04>QwMPz<#**VEDS zx7Ej4c=)R*V{Kum;E13gYhB~WAe_50Q8ff*iS>@KK@mLt2xeGI`3O~IbhxJv&c@e0 zLWy7->_8y;B6aYm(H0KwCN{zDfx3QpE24auS4bo-NCA#9vd}g2!GLdD80v;v z5=>M*BRv8z-jHeZ)(f$hQwTRuLdl0I7=+5}TiZe>0_<>IOjKl~g@OuB6&tFEF;xNY z*qgutb##p!(Eecoo&+o^%*>W(YzSHjjq=o$4-1m>HUq~**H#%0yeC*qT~FYObwd$) z!S)WyC`*`%myVyIf(OpZH#`V#=x*bV@(8v?z}$&=E2OVIAzXzZZ>8+*X(jKjtmGLT zqKZ`wwF(M~u)@nb7@>_LeM}XUfw?3o!HxBU0~Dk2{+6n?b~=~hr?7I=&?(E}}K7Nm+cLIecc;L#?~ zRzMI;qvfJ~@K$i7wUxX{h#A%!7iJ%(=wX5|miILFkyFLs^}Lm^1RXgg|4?P6C^#|@ z7i8@l8K�qUVQIMH&Q#MS6w%dLTTNRqV}tK%3y!x|WbXhJFBeK>f>8AGvMt_dn1F zj(m%$ZYLoTBr!zjSQA{9olmsbj{b5nN^q5)0gqL+l3dxog0oB(F31GGzY*+szDY4AK8#Z+rD?HRUi8kgVdbr837h-2 zZyr?gH#Za3J+3SGve5IXurWy2`VdMlDh zY%*?*TxrIhBi&*1N#!72uH-s*UPJati^HJm0x!EMxG8**4BT{mQ*^D9vfW?=?P8df z_E|=@HnQ5f`t43|`yT372qH5sLE26FfldCM1H_={NDYVS?%&H+PdBYxiFk~ zD6Wi9UAXX;x4GE7kE4~-TivzXN4&P&ps3z>J}->{UYPEv6INkTq{}8@U3!V0CvB*q z$3neTx@fmVpn219uPio|jC9`SRF#*j_xA_`r2dWyD$MKJ%`#;AMqpzN+sbkzZXi-i z+%;F$Qn0iwjqKg>50hsM)r+tAdkqHZvXlBU)ql1aOkRG3URky*yFr|STFoY!-O=cd zY&XDSz+A{<3;HD4_`F~ok7uJR#|LH1nwa%I?v=;N>5_+vG;bogFN3+xU~?M)vd`74Ms7ZXFd#`nK$;V9=hl8aZi}uh$x2ayHX4$NX7eI z+K*xv^%z^<%wdGzWBnimT6l@fTnCySMdsf7S7@CHmvuoUrRj(aYUbU^Y%ZOvY?33@t<*wXzvKUG zkB@@3OJu*{`f_K=3z_(=D$VHUXB-Hkd#1`$FM=;!|^2m%yo~dHd&FhT>CX2Cplo z2X*|3^m|)=!nJb!mSpv!U2^~z7^kcH)_#-!xICteov(|Nu10HXSP%b3aeOY3;?tk4 zQS+tsUV#PKz2@8pNNTd(y@e94HEGA@5RKYrJKTT3X~KWKs?5$QfavYZldQI@_n@3) ze@ViT!Qnzkp#Jkb|L|bpob^3x@#y`P$G*F3Gf(c_x4c7jl}99pG06B$dRm7|7diV?ft;?s98 za<~|byTnLGEKg+hTsZvmOGD7DxfFfZNFjR5=l8+p5SJ7M{xfl5QqP9~gd#`^6WrGMOjy zYzy6f*=?sha%EB>W+&ipWI|hIT1MN!^X2!H98TBLliXhx8ag*K3OA=^yt$#g*);!3 zB-68{>tb#D@3b!|3W1XfKYvWq)bO>OzRInb)`7%6>Lu?-p_-imJFMm!UE{Q41onEA^G*a;oExlA=5X8|NQ!5zG!EccmkoHCt>q;gEp34aJDrf=-1o|eIsWVosyo<-QKr|mwJ^3ytHE}TqOi0EEpLM7_q0qR6?e;!AD-p{@Q&+Z%InJ z1`bCEA7@fXap%C<p?|M4pB925~mldZJTDFUnLEuC3cY+v+aH>dip;9Ivq&{xdbDe(QZ;_ zBa0+9hlb_ENW%6*{PtJf1X2o$7mxGY(j*k06y}Me%V{noS4&EXn_gEC$J{KusFS!e z=`XuO?sk9iG6r9|sj=2c-u8v+3;$Z6{8^R}Qpd}0;!BvGPk27XFQM7xTP7&V$obCk zbqvc|<4+VC&$!JDtggY*M@xBH#z9!7X2L5~s=&rf*`^jPcN9I`GiJG7U*1TCv6qdsWc-F7}DMNR4{#RFjU z)pN$hO9q-4q<&kO$iygV0%yR3=Zz|9)_4WKv9lcKLKBRntII_AI?$?|e=M>t&>ovR z5=$D|0F%!}i}9U39I|w0TMV$MUoTkA=jJsB+RJ#uUI}TmUP_>Ly|lK%41Y0gRiN-iup zp>V$p+iR@-DaqhF3Qb6{Kk-0k_9KN?Il9tf)6pft=Yea7)`g@_yt+#AhsR(Xx(d)V z{>z}SD7&KN%pYf+N;;`h2Omkm*@Tdc%bT2|ZcEzjj9_rJgZA;}--Au_zo##AWyGvL zQ%-C2cCLiHLVlm*@?N)>n^UbNtTnz!>WckdD#IMrK5^}hRs``bfJtow8TOP?^VmqT z0|Pd`mTz+fF-|FhMzG6Y)0vXbsIU%1;w048dQQThPE4oT{{p`9(A}Qm{>iTPtOEpf zyW7`(g_?-MDMs*m7#pCuGl3;alG##qx=Zm?w8~tPQ6rI}@-b-=?9Qux>t8wis{Tm{H2ysSOVMS>c;~{vTPR{l1}PZ_Gr-t zlqL%HTwzyoxXqeC3+%g9uL)HPjeDA}84Z2xW|x{+{#)2os;`w)S3>5557qgZaFbfB z75n7#gkTcl9_8oCFxb?F^GsP@aUk#m>5HAp8g%S-j=i@SPQ~){2qtVtZN&wYcSq1S zaC3@+4R0)VwuQm%8NaKwMdz6T{Pa@_E%#ERXS7sbEb6(K#}#j+fi?8V18azkU#r5@ z^1D(GUyxJ3c!LHu=;qY*_rw+dSK=?(q^#V0>AzCc2<`IdlJ+xl?q73^_3hc)olU@v zSC*W=K16v29GNTX(=1B-*so$Q&kRJmnG6Kqr1ct4lH=rUTWS8dt9I!JYmGB!KTOH} z3ssGOufPS3U|!^>X`%LV>^CwEbLD)Gf$j%P6jW=S+4O7A1E^jqUS*eOAYu=)H$ zc(#ip{7TW~bK4c^y3(}_Vgr=h_ALE!_GyIt=#;KRY&eKwKL^v~@? zxgDkiLSJ9sxQanL$v1URsTT?{@c&Nbq8k`H0(-W9n8#PTq&#UI9D*AIT-A&hVe>D%;bVuoQ)oanoe4dU16j#2zVu;Tiee_eeq4Xf*#;5|Z8Xz=6bPBrF7g*= z%N9FXmvhb6H_rM@QG$qidHQ5qNP;Z75t@y%$Nl~Hoj=EJzF8qqlepC17jvOld#>K5 za?ifOYcP$WjmdEDlx$|=d*}b_o8tBXkyP6+76$C-Sv@}Qt5Ie$m%HIVouvSq((vv7OySfoYAY_>ijjE(!a&!hjPcR${FwutZ(efyJLjA>E!+Ru zn(5lWPs1=~b6c0vZ=%j!>77Gp#>VSdZl`ZK8^}EtO0vHDXDz0P#gxt|VThx61we<2 z*EwfK0dO$sn z;>B`-?X#{2uNs!l=DYx`O~r>hE22=u`1!hLa(8rew}jfZe;$DaM#9D}(Fjb3N+*&owj6 ztE35{5j&jIn>B{>=~3d!EFnkT&wAn9j+%IIwH&FvSN`G^1?jx;X`kwcrxHXrB;?9O zQh9>PUs$)UfdO62^v(9SZWmQ;UGK?9xKuA*&NQEV)B|!y>{; zkt@p;{MhsjeL@jA&>^usm5I(&~3}x`L@~MpqVTtt%Qr3&*fJHp$4%4A# z4^%^5lh3Cce+cUx7hN|(rl_}BnPasD4U09YcH8{Xd$Z1cS1E3aLYots17;JDNd#?c&(h?>H6nIW{Y-2W+>Wm4d)bYi#K?IV5zLgo}8Jz(qAyh zf99p3pW|M3dB7xVI=(G?@&l`VkhwK};yTxMKPTB+GpQ!{w-%*Sv}9<0gB6bxx;#w` zq~fmSk?$#YYxN}50^VHtjiV#=J(EAl6|T`mPtU7z?X>wVladR6pF~(15e1tXWV9pB zvQw1DW=;06B$ryU@sV8{6a7GApk6;FW77EYDvv&tZh%q~V-Fpb)j<3iey^8BK_v=n{cCZEg6hZr$HU_1m%S2(Gf8Nc;g zxT%4SiH)z>3-w3-QhXz;@yheaHq=#aiuI8;;wIm{TS|f4ih+~4X{W?lA{#E=Tdl6V zUz&uN!H3Z!DbzrZS!U$g2TDlB^Oc8U*Z_khdGhF!`| zdr0#a@3Zaqju6Fn%U#^H{eYo?6GYzxUJwd@LKVs5T59}jo?HSj!1TIWgv3->zx3A7 zxx-;F+tmRd9^t{Lm_2@sRkH%hpg+zK*&KSJ@_T?@s!W!HloIU`12EIlIi!% ze&lRu`+1KM0|y+A7V!gjRT_gM{LXnUJ?+WVyOHkd?F~g4pyACcx*%0Tbn8W%EDRPe z%@?><02a~`Vv=iUo+3$LK<8=$rhH65zNm}`+j!kAtIGU!E1n%TRl>$Ocx~s@-FfYNEtPHzGeX~mW`HGDn-iqX|A_C7?NRSaU}OphEvQ+Q{%7C zF%5#mwJvJn``iBGH5y4-7(5-nMq$suWRw3=S-MOayR^eF;Oy-LZ%gL_#`LOft&6GN zCAygoIa(`_Oj4WAwF+BVY5_bXfxVotKC2Kqi*Tv<07zE%LmICr6LxXN^2pP1c|Zo@AZs^NTgWmWbKr>_c}yQTGa7yA_K zs-;o$b9aAEwT>!(iOuPMc!KrxoTQptCVgr3{I4$;7Bsm)+K=K5U~s*K>omygyJLNuHFu%@R zbo(cp6stQIRoPukt&2kVH6^IE9eXio8fpq<_F9TUagb{8V8ow3n~Voh=l5N75^zjuEg`bsgk4 zEb24o;>j7Nahw4aiuTYn7i~Fze8h2NwwqD+%DUeVRJp!nx)$kCT`bqzblZ;U8gvMZ znI@|uXbkp%^_1zn!1o_RA1wSjX(2^(G(}=Gfn#b%Q)JIk3QU1j+4&;WktwM7Grlt* zEI6MqHu&Jz4M z{G+q{G?x~9Jgq{0#*>mS0=9qhHp8cT>Am$|ZS6EDq*9T6Opa46I(j-(WE@QO_eEs?pq-?jGr$5VJ+e0*$nNLX+DWA+NBLGbZD; z_guY>&Bpr}qQ8XNoLx0e_n$Ukao~&}yb8^Q@>5@2(Y=y}|Iz|v2O6^R6;5f#bzh?Y z`dv_MP1)sb{tIC+Z!)Qzz9vN?g7fKPGU1E^H*Z5J)XB7-BL_41-lG0NVsFORc|XF* z#VmdP*XCrJ3pc|FPeiA9yw;;1U8Ct<4QLQQP72M5_;D=au1d=FMQTD|m_{m;km zmEBq6^rO!Z5nOqJR_B@N%+Xs%n74fu-DY0AaIj9D&ZvP@09{yZ-w!&!FLg`pDRn1U|zcR?al{pzn8#qe8$)m{sZn@QI!dxF&2W91o_gA8;%Y!0BW@Kumu+uEBdn+g9N42 z**d!HU-S(V{Rxf%Ki=KfkN7?|7B9oMXRk1og15*BCo{foo_hBki-H<-9+>gjloJ|m^tJkI zJq3^yw*~=L9J}$lZOhIgv>2l{YH4w4`!Mx*EmElmHc9RRvq)s9d^`hr@|T$cL+fAN zi69$!8%P5ry}yZ^b8}h%s*GUq+63yO!)KOds&r(mvajZ&3IxaiNne+++u6Md+hmq% zozwgh0iw-etr<;#nRzGZxfL!`G4L*G2Azx-E~W9z)CNq@Zu2^xGAemSRcR@0D?0C) z_sA{KibHHuB=xZ|CKtM&e(`9EoQ0f5toTsc-fFx?w;JKbE4(M2ov(sPGcS*1ZmF_y zNQO+Jdb+L^hEBLNMvX4fd7`4c*xnCgcVR2`@sWXDP~!d`r*mYpC95?5T##oTZpHBS z-Tz-SCsI1J1fwG~zcEyA7?W{4x7Fq;C^(-wCc{EVH<~oA(?~iN$!;v1p1g(5Cxy_A*R%2W0mo>jDKw}+V|hw`^=_&0iWIH+l-$vFQ1AZf9}amIaKa(a zf9{xsSvZKd_w()TC zmOaIh>)dv6Y+yT%cwScbOEy;k*L3m7HJR)^TzQdnG*&{{ux;S+SOeE7|1SQ&eM@MJ zTwsjHHtZ{@PZH{QCjJAq|6KYv6zDw=8VH4D5DH^SR`TOJLSYYR*IZNnA4Pvv07m3i zxmFx|JKw`5&4#0`2LJJ_*-P@cv8z_f*^}3~xBE{Wujl;ZW>&_dLo}PJ>)1qpYy)~Y zcJF|R{ab#1esi!Mjo7PwIWt@$m&=!dFm|%ZF zPTUvxA6!QoyR^~Qmsjsd*0dx-p~lmtiW^mstOLJjab1HbKNbDsA#cI!Bi~cLI+_3M z*b^wCon7kx4eaNARrDL1CT$^Ze)~j$K!JG3XTC2_;rs!Co$@T7dSp51PJ?zlJ>nJc z6pM+pGHyBvhdR8aS2*Pohkm@hOZ)DjW5Ms86=O&X1Y|~UAL06|Op4^Ctbc+`v>E^# zdr>qwK-Tk?KW146LIZ)5jj5D;zZo9sqpKjY<$GH*OF?9hm~mzG`Toa9k}{qNq+JPI9J$6aPm(h=v*s*=b?UFknw>3tYD;5c$ zO~T$h2!Utkulm{5QY7fn-AK-qidAK^+8? zOw|Wy$1NDBGoB6!p%XWT^%zG0G%x^)k2jPbA&mu?oa7R_44q}L<}+fPQk$`V_ZG%V zb&{$|Kl?uWUou9>c>)kYO^jxJKLjD?n;01e%`J<+oH07lzka^qQLn#!igI7K!t@5p zTyy)>VcZCkW4)dAat6Kx+kvtGedYTsnEp(tdOpWo>o$lh8y0m8uk!Jj$nhlIq6>>hBw~TyWI>h_r$sNqr>X#Dbs)``09+p=i z;b7((x6KwTs<}Izv;@E&ZY1kxxKsfJrBX}lHy}89{lt-~B;x0%B5&tmTcG~ak*!o`8l5%(x4W0dL?665%FbmD*{?GAvv$?bdJcqSvpor$dCKXY|nuL5IJmz=25 zS0FnC=P2Cnd_*6VDGd}M!Ja+RZ4>0#f>l|1Rm ziZ*(?ZyZcQ{3Qf~Nk_J)rxCM4jJ>xXPDM{Ubd{gm=w3FSX=3l1zT;G-Imbffy zxQDtfrV;IsPqX_Rq;u?95az}VWJUPMXE9=`#37|@lU?B==GX5THD&FOKk|C^Au5NG ztSB-v&P<>BIG{AkHH1SG56$k|4^yu${c(|ST^`Cx=l>0jL*|iH90MDv zt6EG|DbWe6``0;r&Zn{5k08q-N?v$L$`%N&l%YiG9|6r2{#XD^nO9Z z{jY`kuZ8-r6ZKyw>i?&vub3wxPZEPZ{QJjueYR%{lp@%ZX9Ba!-N_e)*qbGgNk@D+ z@-l_-IcLl(fLgu~2ZIfbR$4N+z)~doJVZr>SN){@Ex4P_( zcKKMR?a?iJ&Q+g3x+N9YO$b zwE|wSf0{|c$H6|~7|Bqgg*1}d%E;mT*q2vs*JFF%f_$Ex(@{9l5{h+6JnPsjl_%?Z z3rYF+B?*I2=v%`xBXM*0dg_=ApFJFqnrF}TuyhH=z~yyqLQ2uAY4$1p>_MuShmEG&iDJf`y7x= z9zP)0I}EOC>+^LC#=>9RSW(Edh6}lT!X@~|Hgt;V z&>FRiEt`YtC;I8n+#(1=L9sV2jGSbIbe&`-K5W801;~-;)&P%PcE7g$dVz1E8ps~2 z2zXF#zpu7R8TMLwdv^&0eOS48uFKkZ?P)hx>f~27tv`)`sz?N)a5Y%7F?ctx3d=jF z1G(iH)Dj9j0MH$Hd zm5>UBbki#jgRD#y2y~>>j)do={V+T3uJcR-A)dl&Zs|oiLVB0hSl;Lh#33Lm77A{x zJI=?oCsFUCt??@FKa5TBF@we)eCV!Wrp#l&%~_l*@Zy z(MO?zC#YO&b$;>F6S!=Ln>gh*JxJ0P=vh!x9fAY6&UZ$(eOGw(W9JpxgXa$@5Y+h| zJD+2MUDTk;BnG3=3RA4BpiA9}W+3`jJ=_O@I#ER2L&t=0nwo<8=X65Lb1mjmZt?Si zFZ}e@?>2mlk6?+lXcrYq-|3l#!0be?=JqQXgx$OAPB*L08+%ys_`4l}*#`=vFS}7B zLnOXm+BH`tv)BRHJjK}bQI)KZ17~pR_(WOO>HoMwnLtU)zNR;^=9Hxs_uB%L@gQfr z0p>SGvrJTNh$R>{35p&;Q?f_Zv@?PvBMBdqLT||IJPMe4Y9p!zkn~LHgQl~tx#X6D zN4-LH@jryKQ=pOh)A5K{0GhFeMr1rT?V_-1_T)L*+BRmsBr5t@Z!6LIX~U8hz4B6) zW@nw1p7lc(F{8Dl_Z#dQIpw~i6c%1|B#13PimW{}#s1%t^qNKDw8-FK-L!@hve;N-+X?1Zs7w#-XHi1#N$3H8u5~YIiqGvN!Le6v!Rb% zP!ry2jRJ&B2=4DAQYhj##6$&~t~(vircvxFg!W@f7O#<59Eq6#vBJjSQyC>@6|`F& z8zy4jOaQTn80+heZVs0v6a7bo`y-xE@SsWSvz4nr116FE-zITX9EwxS#Cklu^JLl2 z^tz894KT&|9x4kB?AK777DK@V;apWp$qDHr_zgU+U;)grZ{gx)`|CY^;3QZD7*?8 ze(L}#fD_I#9R=cLZkg|wPC>h_GTWoRp^qay=WCt+QN-9SYBS!g14WDs4nglAl@VkC zmmWU;TvlyWd1*0XgdqPnvK{uD`*S(wXb_Jt0UIv5`v+7r ztSxa`+)xgdc_{IaFP2-uhba!wNn4=u&kDGr#T&uxw*i@2{|6X9!vfdys(ewOLK{KJ z54WP9_26MUpk&n`VzDQU?cmX~RMsa{U;FHCK@uGllr6v_1h-2_!h(S7vW&#i>08X&I4;u?ZO2=^DLDdPW~_rZMn5D%Ee{r!zYW|5B(v6=3yMLH!S;3esTBa)|t^GywHaj6YrX=J&K&?*h84%1^kkoHv4EPP)O)rM``{8xXP8b@-V8g*xRuT!TLH1Z z6ZKf%OwFBFew{;-Y8$8JZ15C^`=Ew1JT>F*pZ!s&=x#`3(*&iZ{X&~&1vs{3HM#Jy z_P7{!$~+$>(=xsJS(x6-+^WSb@?SYa%%rdH;$2eCaz-g(gs<4LmP{*lLBkWy=~TRTqy0 znog-sNS@N>29tGZQ;{tIpy!;FC#*pp*LR8?rt%orZ%7}t0-MHuBP|m;3zY!##cho| z@j{=JRD&V8}l+l4jL`;;VZu&(yY8J}P^Ak$-tFgsL+PdwGScZAgTP58yqyq2%e0ntgy> zehC+|dkgdhV96zh;@iL(<$5De@$v(Z(!6&9QnN;{H{9&JNK z*=y8e)Xc^5!rZhchm7;3UcH#;#M+rT|e?!yF(Wdp#&9XJ>(8VMdb;$kYNE z1hj4{$9l<6bJ*g`e=fa}&7@3aSWeKn0`bTabr=dcU3~d|8<1q@iZdjAeYZ zR;gN&8-Qv4q95|BPprV{Ib6R}V2jOj$7mk&ElUIC#Zft@6V4>n55^CEq7{Y?jc>8+~4U#Jcm;GFO{MvXO+ zV&bA3_-3e3|XupnTn*S^O-xzhLEU<%~u_{zv*w194nohsn-`x#o-Hq#K3gdEl2xAL6K(+Ko8b<(Hp(o#U!KxYU)96deC(F z0(HsgLPr$kH8uZ4$0D!?E=V{uo?YeYdW&&cpCNa>X~uoF1c{SH z*2cZk=&i%^AVmxk(zav&mJ6k8Eoav#r!$w7u1XHXSj2=L2t?6?4eOgy5j7Qof$C~I zexQ*0yhVnQH;SB6L=PI+r2h{B9C&WdQU>L6I3L5Aq0sTTC*a8pAvdeOpPzw%1qXli z+&IYNXP00dMYiVE%FDWKsqO)50;Rm`j-I@Tc0k~T9} zKoL5=W4HCFd@L(P>vhRARK`V_1y}W7ndQB zfx684u5m-ma;fn&=aFsV?yD9F6#^7xCheESrf zuUb4S8Xx_1|0qW?l*`AQ{JH=7 z1Np%)B$x*B;euBJbQ4U{pp2^a)%m{%dw4)3i~vH+VEf1Tkm>?>GRPf3@mN~xPoRKN z;^4y%>x?xAI!*##2UTQ25ATVDtB-M<2EA%#f_g3Cjc!j_;h%m286e zvG`5kdlrG5yb^%R8`EFyy^DJqbobsBe_7}+kgep`jJ5!IQR2?*?G*kp(52yz!5yYx zaCGkg-2$E(K>L!hUOV17q>D1yUhw8)Amff)1-C9U%sFidw#|ver4fArTt_^8$ZtSz z?QMdM5g$%{0hW{$3%IIFKt7)tdxH0L?d+=9UQB6O_qXM$VAPTQCLK*eJ)fg}L6#4L zDgpSXuPX%_K{XF|E_bmf3H${ZHibwcpy+pM8|0v=b);~RaGcs=z<@Vm%<@Rl&aBVd z1*d-eDU24XMc4w*y%+`s%K8v!F8Ku!79b3}E@=g-k{f|O#Sap=D+YZkL}l`u;`;OF z?{e&>>+nWkZRg%7e<=r)W!-dV)^Bw$W9T6`zrmAP!kc1%#;;!R`LgcQM9$&&;K?%x zpxD6bY=Z|*gv9EoS4S+2pMF6fyYl`akF1zGiYfw&BHwnmUx4C2P(V;`+MB^gtROF= zRwW6N#MG2i(Q}8;^`vuD0uFElZ0PO@a-Lms#{smE?SwFA$jf_Vi6VONfKU6u*G>czTHb#$;m%L%qH|0n+x|Zt@(^r z&@;XSC3WQh=BzDK<*UG61y0l@P#Z6^Cj0?Awihl2$e$7ed}jeTR~y7?a!Iq71GGCX z+PiKqDUA_ewgwd3m>9h15?=y^fwdZV9u0BY3}1p9YdgudTY^-AUW z&j5?x64U*(_87<>pt`}Q;PBfI+-0Z9z?Xdm(8|jd#&a=|OJFy##j5_v@9+0jZ3j>@ zXI=;nCY64$Kaqn`iaJ-kRHv%`{RT4>{i8a!O0!^l*+J&iR9+@uDhKi9X$;y|s$)%- zzB+1o>C`on{kQhMGpMRH+m@V@*q zf@C&XK|z9K$w&|+2nb4eb8XJOefr+hulv1vb*o-gpI@iS&06crbB;O2_<@PVg|pVyV(c8eexm3pXky)_&ssh3wo2bc?)yp z@x-g177dqD1WbJ>@$fWWEy({K;@p7c{Nn)s1K4x|ZG~Z(rLmV_iIQYn?FpEoh&l}k zWmE-A7y>-lwLubg^>h|)7ICTkP$xAl5z7aEiv}{=kPsZ{6HA{0mo`WO+ieFeeDNq#tw4bFo?J)jR28%BkCA-^6m%kzv&d1F+9eXkVC(mFE3?4F#=m|l{4A0 z1m#MmOc~-xgknTzL<3NgTZrV}n!kNvbr5H%>z*Z^_76*o8)57Ln7~YOv$=h$5A017i-@U;$1ge zzUEF8t6y>ZB80ehm-k~?k@$XA-mf>Pz`xN6TQQnm>X=)Sep(iQDc4&1W!;gJk0zxu z6^A5L(BND9TU-B`_%{}JiMD+#=x>0E;)2Lo?TUOayGVJ)+KwWdQag}!$G)wwJ%NPk zq|a~n<~FV2Tj9P>*wLkG8l8Mbx zsv@bwL@ml#U_EwU_0j1~4WGHmnR1VldOgo;f3hP5{-4nl#6dt%QF%Pm71w|JF`moh z)tD>k=fh`p_dZWr8dpl?9>Jegn|GsAA8d3FR3iM=|1PXUU$v98QcZXmXWPUZEm1;; z_xD)?u`3Tt{X#~3*u#@yYj37>i9RwHD=0;}zbn5uiX1Sb5j6z|v=M@Nt7h(W*2nh+ z$cjyx)RiLDhjq+3F^CSfqBSnkIB#Tz^37#A}BVI*HMZjHPkuIiJ^g*qwDXOoiQKlOe1B-)de_d7O9F zYKDFX!%vT|A?+BPlGN$?zncUUmH)d@YP%b-2hMj3O9jPwaD=85RZhpelX)g$NAY$L z;>d$V9VQ7kjw>0`X|e#;;e6SK+?6PdISPcsQ5HJcWC-zSJSraQC&>u(DY;VChp+_8 ze1F(tEGfT!NMq8;cKdofaOVWV##Ku?s|5Q~HbAkxlyZCW5Ub~cvT9Xp7i?SaLa1vXW7rq4E8kjVem7Ibtkux{>bB?h~RrtZdZ&yDsRRW%FI0O+o;0n43b;K+PAX5;Z zvw3y^H$9;;w}uq}nZ|6E`Ktwk8gKq?HN;m4f=js`fV!_h9V-{W^d7|XZX)nEzo4td zTDaDG*F4UrFJA#!6p=iHe})G7M&$lUwrUWD*e8}jinB3!uad+6(|0I`qgU#HczTc2 zzY+wm`7q|vogRz1DZTpU0j-ttv{QFMd3bBY8!)jpXX`dwuol<^u&T${wN44B+qRZ? z_cpq$AXsYgUM^|(U_qP{tBb0>GW!8Cb`ye_o42rgdh)MvJAliBb#%q?JIm*~aQ{== zipY!+I<}>9e&)$K9@^{o47?NYt2Jf<8Ee)H22l}lS*;fPvo~d&9 ze3soJh0jodk)uR9_+kNYF|U`}Ua4r+^#9h5{sj^oN_rtsL` zm84_>?C05cMZ5>9pFU@}$^jWgOM#Ts_kk>+H@+5;$9Vv-mO@T{gy5YqWj|JV?(Vm4 zMbFFw&UlFse1d66$tX!F$ld@7CI@wgd-z=ds&_oI{v-&NVu*nrVl+svz&Lp=MZc)w zz;7EHERe`#r?6xZq3vK&5nGhRBqBrZMX*$`%*a0PJfa1piwpIWrPuF4+80IwQrUY{ zo#02Qi4_0A_U98AKK6diQV1sj%#@1uos>VXb{1Y7Bs^{LY@PGg@5!@Nl~jgOl4l3F z5jtgJ>?7r5VoVPT-yF>;yq(1fQr)0&;A*fb63U`8i23R)f?7FD3ys9+=@>$QB z2Im}m_#V+4@2F&tZ^>s`RN*x+@ao;OljYFCNe5}Efm!e;5H*v;#)Cbf$A0P}`mM(Y zJd0;45b}jOLhN9uWqETB)l4yyXI$=}1R45Qn;5W7+aSZ;tAQeXK47O#_bY_5gStcO zsKC{13hx7zdbPLE!=O4*a70=@mh8ykPsMA?#^K4~wgFH%^TiJq=NJA^`1+intlvs!>kTHRwab-Xk~gyi%WtueZ^`muxWD0Z`a@ej zyplK^=d^&7AZ(Y%S-h|VL9Op3jP`E>G7HtOP#Jy!AH0E%4he=G_vh@Dbd=nW4UfT={B5_8x^5`fdEgBKU{|z71j5cMh(1VJB|(^N9?I2eZ=BSM^5u; zQq%nI+%#0=y0{P#sffu?C20?SBMmL0><|B#e&n z6j)I7BbIw;9mjAK z#gkBMwoX|p%AwKV+ziO&J^`7=W316Gbsx=#l^iT|DRh`<$T}bI4nH)0aisBpJPM2U zgOEis!WPBXm44_A-WTVu+X z>L^JubYII|I&YcrC<-U~R*tY>iV2@{Pvh%wAlsUFDs|iTSa5CxxVa{Yhnaz8>pgK> zegyL|^9IJTw*|VzR_DpXYD3x(ygO$@;$rhve*-S>d%O`e5RG!zRx^V4*QS~wY`!|Z zPfEqkm{Sr=U11=M-XPyjvDhV0KXtJ&Pwhjd#f1fM{CpwlwY0{EWlIW_pf&}fEelQD zWN{>Q+TO4I(PiC(lsL!d*8x=OeV4%L+kJ^cR){xP<~S8Mn&i$!EpA0d*aYznEGuy8 z=bQ--+datl!Yk2Egvo}WCVnemtx1fL6CbmP3lK9H2;m_k*-kV8n4^}7?ze(g?;K8* zQhiAeU8&&^VTcUdJ1N^B)P1~k+u$>{cRhW%metigzCASC%=Lb+q{i8ahv~YsGUwg_ zmL5y$>tl5xbM(u_pRD9C5}#5zt1Xn!k>>24pr4D!k-myo_^hPDhYx^oHNAZ{Z7k{X z?DI4M<_~UhI&29WV>Vx#Z;=@(N9$%dCO#&fN+gtv08F`v@!p0wxZ`3YGqLJexnCMT zMnd#v>r%J+(O&JOq@oo0FNnMj5nTPBi=X8o{LWL{7mhoUdHpV4NdxXS=oSrE%f*aA zas)O3&QC_UevN^vR^l{|y_UCzqm3ia9r131`<(#Yvp@?av|L5x8D z;owFyu=!j7lh?i-u}y@O29VFX`FDZcsHz1=!F z``8?5lS!=kejEX2PWAX0?ikSsN_ysHuboo$BS+q-E@@WZ&W%WbZcB?D93F{o^i1Ga zA?*(qGANACFgBPYKm6ac_}D4=RkM<$nDt#QP%qc)oa{-yf)N0Ag>{ca?Ej&fvRYMC zdA0Lt-g8Jzy);l2yh$(;_M4baO#8EjMD>@VjxkMkVA_%W#aHRU ztS360Pk1ZBR&g5U(fuPOyGkbiV@1q7Qk^L0zjHK?9CQB8Zi4C7zaphjU-?HmOT_4b zYbgaj8;l+>A{0K%V)@`#TsVBBuy>ppW0!u{q(|Ae!-a3^Ft4?+iO-!nc&$98VVBP(S1dG9H~$jW#4$8xF;Ss&7+;A*rmA_ zpA8-z7m$}jxHz}hvf5YzVBE&)4ihWT2P2nxBmRR6FgJ1S?(|Q{Cf4j4S9hegX%lZ2S~2iTvN#*XIR_Y;}yc1LPkjgj-tr0W>+X}!O1*E%Et$Kv5E zon=M9*hzzO2Rg(y-2aiFx$Ha-1dRA9kO!RE;sJ~vUj;zIGh67O)k_H&3l!UbG$-Bh zN`Q!9RAztdIPgBYEAXRKBJzG7%s34{@kU^ZO%( zt`l}!VmpN7m*w_1X^taWMsO%f1AZ)nSq4}bQyd2(X2%62Rci{~r!-nt!0fZ8J^Kyf z|EMz(Ic$MF1N_lx+=jbwLqYAHHsE%W3z?w+4pPp}AoRmaO5u3AHBN~6(J&n#Nya~( z2et-b_5&-Ti|L|K82NB*(IF`Q5&!C65D)C8g7$DVTLQRDFWqO_5Cx?qsL*ac3)nu# zSbm;P46`h=w#7$(dF62$&u-SXa3I>|nij0=V6aGolZx%U^VKxraM``JOUN|@#91?7 z`JIJ0Z$mh*4CNw;iLK{c;K78Lef+6NjaVi9=hh-sim(=MA{R;k9{39qK_sBb0*C8_ zc*t!s-hY}tj$^sjfe%IU$_HG3hIfWHY^IvRH=9TWKdB4WwcLIP^|xfC)_|0aWE3Ms^aq)z=Q_(xfj#x90_jQHpyOkl_rAAs?&0^~cJv@+zzN|Cdc ztI6x9`mVaB{a%W<0iQWO^{Bb;H=0r)=Ulh_gz%ve;3439_*_+7Umb$lp`PLvc6qkE z7j_;6Hr?W~UeHr4ZjxzIv(3*?K&pm7+#aNwp1=;};&328;ebant^29IU%=sRX`#82 zO2xZH-{^~&(=r-c8CNwH-Q|N6y{dtzeiKsnY{Yo}f-AKy4_$Bw)dMq7lHr##3?Axw zwuILrOasi>d0NQp6gA$)Oe2DFfD}|xFnbfqjuU)K8=^j^CU55Mf^iB$Ie7V37~$^v zBqA9GOr#I0HQj-&9~q=i2cUzn_mqKn)!qw(0*5a7$=d!au>nWJD%f=B`&Z%4<@Qi; zVU9ZfeGBH={0{ATbHu|TL+JuQH9O_&l`HhWl|g^^Opu~dhM^sxke$z)klSD8m4tLr z5#h=H>|y8hT<7;Zlq#PlfJ$M`FKB| z{}s)q%63n^=q*K1(I=6XhIJ2@JEC>!c%6; z;<`z;GC4gJBcsYa#>Tf(R+%4)FDtbt>6^W`!aWDNM;*6XsD~z?CJ{P2?0If1N-?xk z=YczT+z;r(*2Dh40RmdTbse?KfZu83^VKuDQM6;5(xF3`N23l=Rc)qZ>!x zapQS*&t(EazI=ohc)iSC+d69oYJT&E*tM_NH6>bKx4>4~DME-#NHJ?j9*sduNaI0a zz0P7E8j?oz!6Wl2g06h+e%39R((I%NIT}(F{>qf_M;UTg$uo7mapMYrN{b&vx^GOl zjYC;W9(J~m`LNv`eyv_NW2#j-U2f5a8$e#K3B-5S*gWODbHBz0x`3?o+&a^443Il!WZ8tKrI8RTO8{RoOe`ON#xwSbS z#1p2Nco26>r_!=fNg7ZzT14M2M;yZurTm(3H^PVW;Ym~Ugz~JoCzzH|D=Lrof+0u! zKEqq=nSCLjcW~--z69wz9~SBqZetHXG4!T8$6F}PG$NUphAe$?G|gH$*dGD8IwYVq z^U3$P$|%yTML5oKLC^iejP^qafM0C815b;tl)>c**Pwv>tnwxieiWHM#l5u!Un1_> zI)?L=zj;y@vFg{3XK~8;Uvp*;+_8juT(LcDrqY*ida%bhzOH?p2#Grq~RxCK5GXZHW4 zQ6F%NI22+3@!Sq0?uP&26eX{(!=d(+q zmh>}75y_E`n@Cl*%L&#I|x#TRV zhU6J74NrdAAhBf^+L?Loup`$2tyQ}(y+Kq5s0>pq#ZKfH9)2eunzU0M%xd24)^hYD zP?!^CgM0VM6(}uFbFq#ole_ByFH6Y*;au&*hHUcAv1R~sBjF@|w#q@rD$ z!c$2U3Ae2+TnWP(k#eL6pMg<_q9>ZZRCMT4@ThNSTn_q}m({~F-#dtt^#L-~pt`tY zep-Yrbq)y~1n+1paG&E0*$LsoTmFoODss%tm~G6|kFL^^MERw{^?FX{&+53_lQ* z;BmKVLUl|?s{6YJW?!1yLbd@eWo>Z(82Bkj2bn_jzmu%vB+>GN#e~nmwvifp+sz^x z|F?2k*nRoawH|ImUDz6Sl1=mR0}G5u&9c4Ms<0+)17Dq50%eC4D!wiM=&WwX$WDKW z57oOz`sgjsYvR^7}bVuTn_!sY)f0oX50?}O+(KX3Au&~gt2Ob-9s0JHALz;9+ z>f<=W8)s$jmpU3Msd!f|8WJl&u`MI>Q7f~W}GYjw2VL zh@E$qmc=9KEZC9mer*HB;8VE+#7LDc1Kr(j!tv1<)O6uH2vOK%zu))4AOU>EVIX z=yUB(UpqtaNp}*aB-V(&Pon32K1;IengQAoSSCS1bWFNUN&=WfdAIu%TiOwxDvRB^ zPQ7QVtyws6bWFTZPk;3jr1QSt@Zj?t(XF2>ETv+XKs0dxnRi@31=FpHHEPzLpj&lr z41W7r4`hb(=yc2VBI>;sDZ@3XK!@QAI8*^Xp@JEodm;V|oC^Rd>)B%Ak99He8pp2v zZbA5$nkLg$Bs`1$hLE^5>^Q@zd4bU7M{Dq4%?vu-W}QY*p?TG3fXFPP0z#i3+V%ds zCMs1Pb3}v$BmLPxc&BAGU_NVB>!V>9S78`MQJYD5$CHXX%+zc>!P5+%2rm0bb~LKN zMQZYFm&0iU&YF9?Aih5vp+y`IhMU*iUK^H1gy6mybG z{sU6C;VJH==6NKgKR-htw|>XXf&ho`fFf|B!kOhT{#oNH6=CB9B+*Q)LE={MF0YVP6@F~NxN6m;Hhr??RN#2JhE9)! z`;Jo8Jr^`o6p5p!z87BpT5;pRz)o8Wjiq_9Ji|> zzp3&hgfw{+Y?2GNBrDXtiwu%)BJVY%O$#5jOOd>Q;o1}1@ki=lpxJjp!@_(e5( z=ji@q%-Y8K*B)DkaCzLnf|kOHzY_h+h8c4;_p_?S)-SkiGzYrU!fBs73h06Vc zGe?GAh=Y(e2H9frGGi$PQ!&WWDhOU7elUMBF#aC^-w=Dte|v}>IY|Hi;UOmS4{Xa` zAf{yi@5W=`oO^_sFDk9;sAbSwYh~vY&}6Ila$5maK}4b%%9a}ld!#`h9!!<{Lov# z`y2P32)O^_lk38xOD8A{=a_GF~y} z(K!rU1ZM!1%C5+4_4n5ikXFO3c`NY5Ky($DgAd&k>9U9WCyD^gxa!@;WEa-3ZWqC1 zi-N>s#P(T$w=w8`SAgAEo}_2={;Jf9s4-}wRsh%{VT7{y;e#cBG2Vt!Nb-!`0!A!( zDlksnM?u2moE>-mFjh{r*wF-Hh6x|)ushHkq&#ewR4;;-DiSmnr*RF^qA*72F^yS|xXW}b9>lqJQ z#q!$%7W8uJV#5dF&LNlT^A7SWTYz@#e9(f`3t1I_y^X*nhW09KUe|@#&3Bp@P74PSa_8MVMv|` zew0jxkb2ulBHnp5UmrUV5T-eY~zE!wBCy;Xeo%Yd<~%qaM&uK+omg!eS4^%4SL z>atO>{0bd+1y}BEfIY*ET&fpOBXhwCl8v`8hCj9;peDE_`R#RSL{iqxytKB1_4{|j zmXS$wrGI8xGQ zZ&C9uvxB*1IGyZg8pagpN$EY74~^97e3qpAWm=O=d)R%n*EB5U_NFKaTO-D;!}0fr zfE}|kI&t6~)f&(v;`9US2c-YZWjHjqfO-?bFO-@|EwH^G9`~ao8uqF#B7ITJUX_L2 zEqaVy(R{Zg(f93W>=e3(=9|(x48GjANG>@dk;6)*K|XbQRo2r?CoXft=>O zv%*fv@%lztI>p}U`x8;?nXOxMf_I1ETEJNg6Qev_xx?O+dfM;U(oZS(2I#2*yPS$W zttHote=#)dw60P2H=uDy{a>YDDy)zO*7);!;3IR_QWRWzxf;b8&$u=}B!JB@iCyAj zar-gr7-1eRuWGu}n`qO!`&$*TUyXt|fli~@LZ&g9zN;9Evf@7KO23OFLbgMn>xv}{ zV;UN{(*Wkcc+J=%p$$DOwG{KV05Mde!o!NKP6r~EPK_P#Y63_9TWk%#zpS1<$L7NW$dXw_ zBLPd8R%$E&?qaXWpgU$;rasZ>o1A;KX#aGP@m2VxGzzENnX;*bCO)Wa+0^*p!1xuO zREpVFhfm`5>v1TxPrl{Fo=yRr(P8_TYIhy06n+ZBs3+X8RH}FTf!RYRWfmkChwH=n*Wg1E~)-`{a zF{fRMO<<>|xkAWpFEJV%=_Vd2K`3>3WSHWsM*HO}gl3VNdu0{+78J6-7C1o(vQMzL z3;GTj_d7P=npXj-B*F*< zztK#jQ3bExs7Vlp)KAl>888`x-m}@8{MS9_+{8a-ZQOsEwL-^|o7^R+B4tWw@6X*r z_OgZm>ziy1T%g% zyxnzK;#%;fC0L7`B&XEdE}W z%Bb^OQz6lz$BWCvegxOB`$T{+N;qyQ`G8d92WYM%?;q0?|Kf>j6{^~}OZ{XQoy=UJ z3kIunWmu(Q{3o8!HKs8$s%=6rjc2x}}>TEJb)nc5AGzNvDNCb~G} z9Id=Mxf91siBs$TO|4V@+qH8(ZtZ4E6lfhsDnZTX8)NuLM-Ru8 zZlJ)dS}kwVf$^o1zA)S8%SPc8ZSoBfrlWjbaM7Y)eki~0dtt5Eq8pM1N<2jd1ixgK zg^#zYanTOA);;=|{K}dM^M|ib((3c!ULmikf03>6UNkpFcvwf554U_lEiY6zY&ym! zbJ228JTaN;Y2KC&z45CEn@oDs_nxR2Et3+me&OZT^Fkh$z3Mlbcvbb0mz_OU*1`HD z*eJr3d4UqQSN*O}zv=0tzq!2oSwyQ7r9ZJCdyJ!QGy3uuTWSJ_o+ORPFmoB)aB$TK%7{*7mWgECs4zgZT? zVKWh%@qyW?*waTHapk4T&kr&1ewl(i2?E|a374yFpOBZRw4J2eJP?8p_!-InMaeIL z;IRd!DWcEWzliV>I52I$emu?idULw+0@I}eq*Iwm zi=IqX@#1y+kYFQEuD+fyKhN&hj4y45N*!XXQN6JbEA(PeG7Zp_M# z&&bVUneCDnXg+|7{3t||x1^R{W;?+-R{?jbe1xEd<~PKq&LSU#a6d-?4SBo^SQx4r z?aD09HvpQye?@Ag%p3BV;nV3(R<}Nir9c)HfTTx5P^nY-a;Gx0uwta+{J4+xD8ylr zr2G916A*t_j5ri%7uRfb@Lq}&W}`JF@o174SiMx&3loz!)GuJ0Y*us`GKMn#iXp3Q z;4l!5gGNz39ioAiRO|TXzfe$m9D(AiO!T4ze1|je%gKtL!F}1wP<(z1CEQ_zwr%XU zm>YUxv3(0Pl=vbdeHtQOUNw+fbq|=|4X8nUuLmu;2*o-)wLH(fBgVZ@N4cetrlBIZ zyjZRz^q>!Q`!R(R7a||^hMMmva;pT$3!Wp{j^*48@E<7m5y$CecwGtHM>A?6R0O)+ zi*NNuAP%?#nZyuWs7`yYUG8vKQwq-Bre=TOT9iN!_90Y)QJWM~Cx~XmU#B9mUSp;I zR)Kjrq$GXpKwkdcX?|dJP2*g+bLq$P6<`S-ne>QUeC!isKj+gOUga+X$)Do1B^;j> zKxPoCB^W)0WW4*c+;Syw`}B#uVarH1g*YhmkpEiS3T_i081`MppM?ta|`dk92zRgQ`MPS?_ zBAuztqIb~+qMEfI!4>)==|eXqQPb{_WQIrnCSY-*`hXu;0W|zkuy(eGPIna&8Vd1+ zE2%<{VbME9@>NcTRP2fsNC4jG>KSr5#j%1yVH>2u+^>lyn_Le|yCe%B)I1b% z(%jo=m-{~AiI&#@IKUf?O*jPLfeTzvq&U=N2!42>zTmu_(kwFp{Suv*-8aPQGZ}0- zhkM!@3RFondsT3g`U)&6N5=u%w995k^}LJ}03SBcsWULkc7LHn>m9>OWi29x^A z&Z&2NkgAsqBG-X)=FI@ToMco2k)BHQlVpoLSR8`4tspUb)I1pQk=WvvBbU<)vngrV z1SrjJ%J!?@z9H#nsaeb)68t10K&a0hWE)|+6i)`x<&a2*m$E|jry6lP^X}w->**}H z7J|SWfoq}1!|37%K0jJSZ+ASo)x{)q_p8XWG7gKeC zsvOI70x!qhziekGYUTk?g(_IjSi``+37D6fATD4k{0282k=UJ}f{3nf7;;S(Jrb}L zuR$3F^GCrYEV52lI4bs==!q-DQ3@d<3=nY#7<@#Z&`hO}o&+u8IO?;% zMMoDU=7S@pyG%^nTj{1Ap4qv)>{7Ere~+Xh5cM&%FcESGL4QB|s*=wkhstb`Y~BVE zj4^{V2SXK&7q6!hu|8_n|0IM$9RMTO7W z`-~|8J^kh{Du-sLad#HP>&fCS3jG)K>#fDG+9 zS##O#JHDOG+LseT>^lYcl(32$)Io_+@AgT2=VC0HQfkG-J>w~rq701!s z(qG8mYUgF>>M{TfA+5zKs~R%lr(V03pD-YvMxkroFeB{|UF>)Wkbx{d`2<&pjR#-c zoq>xDcUs1~tp)2Wnr`<@#{MR#UQ2LK9SMdEs6+kfGvO;UX&+VX77TlbCG6&iuihXq zc#3jQnBp~UCbQ4YwhZB|tJ9Mdc98!4wY$ycdUfV`G*$u4`6BG7Y8wRt&pgf(yZA2# z(&Fzu?yTP}b!SQvcu5WrWLTUdR+dZJ1MC1lF3c}p=nFdt{KYe@2sUNdqFJ=;{Q@*q zPRH3y>dmAJSz!Za1r1_88_iRh%-qcCg;{Y?^2KI&(~eQh=*aT~wG)5kg5 zt<@F-l${Tpi|s_-CLrKv^P>Z+fZ=?X^V44$6vc(diVQI5Pis@5^i2#>o13p-Sa*~L z=$Vt=uh9QZzEIn9p(A>tIO1e2&Rx$JZpx zH;+n|*G-p;HPOXl5-5YpByMut*AzpQqPjc<;sgeDDEINDzU`R7cp`nzBcz2PGfbNa z0gjdMi*L9vS!@CLz<=HD?^5-@{1_Un_kV{G{$&^1NY+*;R$7T*Qht_(jaD0^Q49=q zZ+zkZ02-^f%64jNgn25a71Jobv5tokm_|K5`{*+ZrjK2>%Gd1gwEh_l;nk)lK$1As z7%iRKM@Tl^%N4p}{$oW$N4?xu?C*|gR;yC6lNqL2RfX3^dH)LRR=*bh#?Kn_Ifb&$ zFfXFql4&Yx2t5l!etk{tYci&f2NmFUh=2Am64(#^me|?OWMS)Pox))6iL}pi zzB3}dT*a#e&ruKKJ&%ESO4fPJO)!me@8M0?$BdQBxBiPd|DHO47X5qbSj_x;>U5a~ z|9jW@0}%eJE%5)xUFX&pGfQGNxj<=_SCwi^XsXhb%@$!@mnWM#F0R+~^kR$NjF~a1 zF+u_wTUh^V=}dX{82&j98M^)tM7fs*k6kVjQc|rx+vW-WCT;N&(?l%dE>D0S^-96; z)oh?Jhs@T)j~lC8b&n=yXz7w)7FcPheE)572Cwi*s0fFQD%%$!jyligWGh1gwf-ax z!qI?M-B$VNW;0z}P6pmO#nZKko?J_>82hxRg|BnnntP+M_}<2NEF{rvtm3qz1?Hc6 zbo5jSUEX9QE1=ai!$pO4n0nPuW$~(COGJ<@n&OU?_~68dx|$9;8ES%@lmt}XAX&RS z=A$1~l8HT`+FJbxEfQ;Yj=r62rnc#dq!KoH8q|(<@wokLH#*3&7zSh_IsPt z^-6{~e3*Yq&Qg=h(Q|E7#?y=T1J#=vZ;4Nm3QO zf_il)`6VEii$-K*Xz0u$W%2f)&0f~GViQ%0rgYYlY9w4K zonEaU-7R+1>c$}UlZh(gWS0fMpmGh8gA+ZWX*}*sF?2c8zu?RJ?_O1}&)(GWxZU?z z?>~A`mD|fEIU^_71(3;VOZ5^yT&^X+Q&;MzUs+|gSQftuPKZr1J4s`vT8-%{c;2hT zDi6cF&Dl5nE*Dh=U5p=!p^kd-Oab(wp{DwY$=nwT%uM-Ye@qa}I7jQxdBX&39ygtf zVJF7z?YwNz=NMs~dFqd`VZIvm>*rS(>&G({EmG=&T;5^Q9!Recc#SFol*Iidt zbgDLmsRuS6Paz#9%+~B>dT7Rl%SCY~Dem6&vQ6sfg^u1wm?an8sg?9GrNASXTYtrk zQrXXaB8w0EXf>*9bFo#v_fR9j<(Huw_!QBZV8(p9vUfBE#yeh3>Ghin1ZvFQMOI2% zO)%o5cl=f?$R6duI&NR+$#z~IDmf3f_aIe%)Xz)cXC?2q6kWe$##eWJaDxXw<>=*v zE?F;)R0(d#`y~ohTgv+_M)0@3Av^lA&SP&rUVp;K7=yj2`fDKqW<5&Z5jOaIOZ@!G|JGlNuO#}k6FD^my-qy%`%zt7 aI;?r}MhXY_TNM`kr>b;Su}Hxp