From e4606855cd47392b79768f2584e38a3494eb35b3 Mon Sep 17 00:00:00 2001 From: Martin Chodur Date: Sat, 10 Aug 2019 17:25:18 +0200 Subject: [PATCH] feat sicedar: added readiness prober Signed-off-by: Martin Chodur --- CHANGELOG.md | 1 + cmd/thanos/main.go | 2 +- cmd/thanos/sidecar.go | 29 ++++++++++++++----- .../manifests/prometheus-ha-sidecar-lts.yaml | 8 +++++ .../manifests/prometheus-ha-sidecar.yaml | 8 +++++ 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71834004b80..7f6f547b4c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1358](https://github.com/thanos-io/thanos/pull/1358) Added `part_size` configuration option for HTTP multipart requests minimum part size for S3 storage type - [#1363](https://github.com/thanos-io/thanos/pull/1363) Thanos Receive now exposes `thanos_receive_hashring_nodes` and `thanos_receive_hashring_tenants` metrics to monitor status of hash-rings +- [TODO](https://github.com/improbable-eng/thanos/pull/) Added `/-/ready` and `/-/healthy` endpoints to Thanos sidecar. ### Changed diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index a44ac436245..df233e65636 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -70,7 +70,7 @@ func main() { tracingConfig := regCommonTracingFlags(app) cmds := map[string]setupFunc{} - registerSidecar(cmds, app, "sidecar") + registerSidecar(cmds, app) registerStore(cmds, app, "store") registerQuery(cmds, app, "query") registerRule(cmds, app, "rule") diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 1cd95bc44cc..3f7222c33de 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -8,10 +8,12 @@ import ( "sync" "time" + "github.com/thanos-io/thanos/pkg/prober" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -26,13 +28,14 @@ import ( "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/store/storepb" "google.golang.org/grpc" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/alecthomas/kingpin.v2" ) const waitForExternalLabelsTimeout = 10 * time.Minute -func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name string) { - cmd := app.Command(name, "sidecar for Prometheus server") +func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { + comp := component.Sidecar + cmd := app.Command(comp.String(), "sidecar for Prometheus server") grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd) @@ -54,7 +57,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri uploadCompacted := cmd.Flag("shipper.upload-compacted", "[Experimental] If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus.").Default("false").Hidden().Bool() - m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( log.With(logger, "component", "reloader"), reloader.ReloadURLFromBase(*promURL), @@ -77,6 +80,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri objStoreConfig, rl, *uploadCompacted, + comp, ) } } @@ -96,6 +100,7 @@ func runSidecar( objStoreConfig *pathOrContent, reloader *reloader.Reloader, uploadCompacted bool, + comp component.Component, ) error { var m = &promMetadata{ promURL: promURL, @@ -117,6 +122,12 @@ func runSidecar( uploads = false } + readinessProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + // Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes. + if err := defaultHTTPListener(g, logger, reg, httpBindAddr, readinessProber); err != nil { + return errors.Wrap(err, "create readiness prober") + } + // Setup all the concurrent groups. { promUp := prometheus.NewGauge(prometheus.GaugeOpts{ @@ -148,6 +159,7 @@ func runSidecar( "err", err, ) promUp.Set(0) + readinessProber.SetNotReady(err) return err } @@ -156,6 +168,7 @@ func runSidecar( "external_labels", m.Labels().String(), ) promUp.Set(1) + readinessProber.SetReady() lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9) return nil }) @@ -176,8 +189,10 @@ func runSidecar( if err := m.UpdateLabels(iterCtx, logger); err != nil { level.Warn(logger).Log("msg", "heartbeat failed", "err", err) promUp.Set(0) + readinessProber.SetNotReady(err) } else { promUp.Set(1) + readinessProber.SetReady() lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9) } @@ -195,9 +210,7 @@ func runSidecar( cancel() }) } - if err := metricHTTPListenGroup(g, logger, reg, httpBindAddr); err != nil { - return err - } + { l, err := net.Listen("tcp", grpcBindAddr) if err != nil { diff --git a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml index 02d37d2dc5f..a4362035e21 100644 --- a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml +++ b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml @@ -145,6 +145,14 @@ spec: containerPort: 10902 - name: grpc containerPort: 10901 + livenessProbe: + httpGet: + port: 10902 + path: /-/healthy + readinessProbe: + httpGet: + port: 10902 + path: /-/ready volumeMounts: - name: prometheus mountPath: /var/prometheus diff --git a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml index 9e30120adec..11b48303607 100644 --- a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml +++ b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml @@ -129,6 +129,14 @@ spec: containerPort: 10902 - name: grpc containerPort: 10901 + livenessProbe: + httpGet: + port: 10902 + path: /-/healthy + readinessProbe: + httpGet: + port: 10902 + path: /-/ready volumeMounts: - name: prometheus mountPath: /var/prometheus