From 00e9eb243975371c166c6f0e6cc7de41d06e3962 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Tue, 16 Mar 2021 20:28:43 +0530 Subject: [PATCH 01/15] matchers support in sidecar Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 2 +- pkg/store/prometheus_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 7239c4b3ad..ce2b905db8 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -496,7 +496,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return &storepb.LabelValuesResponse{Values: []string{l}}, nil } - vals, err := p.client.LabelValuesInGRPC(ctx, p.base, r.Label, nil, r.Start, r.End) + vals, err := p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) if err != nil { return nil, err } diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 4cda3ddafa..38cca22e5f 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -441,6 +441,32 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, []string(nil), resp.Warnings) testutil.Equals(t, []string{}, resp.Values) + + // check label value matcher + resp, err = proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ + Label: "a", + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "b"}, + }, + }) + testutil.Ok(t, err) + testutil.Equals(t, []string(nil), resp.Warnings) + testutil.Equals(t, []string{"b"}, resp.Values) + + // check another label value matcher + resp, err = proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ + Label: "a", + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "d"}, + }, + }) + testutil.Ok(t, err) + testutil.Equals(t, []string(nil), resp.Warnings) + testutil.Equals(t, []string{}, resp.Values) } // Test to check external label values retrieve. From 51a162d85ebb3a1c272a0129dd6b392b8240d425 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Fri, 19 Mar 2021 01:35:27 +0530 Subject: [PATCH 02/15] added and used buildinfo api Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 50 +++++++++++++++++++++++++++++++++++- pkg/promclient/promclient.go | 28 ++++++++++++++++++++ pkg/store/prometheus.go | 34 ++++++++++++++++++++---- pkg/store/prometheus_test.go | 17 ++++++------ 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index dd93addc85..8a8cd3057f 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -208,6 +208,33 @@ func runSidecar( cancel() }) } + { + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + // We retry infinitely until we reach and fetch BuildInfo from our Prometheus. + err := runutil.Retry(2*time.Second, ctx.Done(), func() error { + if err := m.BuildVersion(ctx); err != nil { + level.Warn(logger).Log( + "msg", "failed to fetch prometheur version. Is Prometheus running? Retrying", + "err", err, + ) + return err + } + + level.Info(logger).Log( + "msg", "successfully loaded prometheus version", + "prometheus_version", m.Version(), + ) + return nil + }) + if err != nil { + return errors.Wrap(err, "buildinfo query") + } + return nil + }, func(error) { + cancel() + }) + } { t := exthttp.NewTransport() @@ -215,7 +242,7 @@ func runSidecar( t.MaxIdleConns = conf.connection.maxIdleConns c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, thanoshttp.ThanosUserAgent) - promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps) + promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version) if err != nil { return errors.Wrap(err, "create Prometheus store") } @@ -348,6 +375,7 @@ type promMetadata struct { mint int64 maxt int64 labels labels.Labels + promVersion string limitMinTime thanosmodel.TimeOrDurationValue @@ -393,6 +421,26 @@ func (s *promMetadata) Timestamps() (mint int64, maxt int64) { return s.mint, s.maxt } +func (s *promMetadata) BuildVersion(ctx context.Context) error { + ver, err := s.client.BuildInfo(ctx, s.promURL) + if err != nil { + return err + } + + s.mtx.Lock() + defer s.mtx.Unlock() + + s.promVersion = ver + return nil +} + +func (s *promMetadata) Version() string { + s.mtx.Lock() + defer s.mtx.Unlock() + + return s.promVersion +} + type sidecarConfig struct { http httpConfig grpc grpcConfig diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 58fb296a03..0e29ccdf5d 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -607,6 +607,34 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode return v.Data, nil } +// BuildInfo returns Prometheus version from /api/v1/status/buildinfo Prometheus endpoint. +func (c *Client) BuildInfo(ctx context.Context, base *url.URL) (string, error) { + u := *base + u.Path = path.Join(u.Path, "/api/v1/status/buildinfo") + + level.Debug(c.logger).Log("msg", "querying instant", "url", u.String()) + + span, ctx := tracing.StartSpan(ctx, "/prom_buildinfo HTTP[client]") + defer span.Finish() + + body, _, err := c.req2xx(ctx, &u, http.MethodGet) + if err != nil { + return "", err + } + + var b struct { + Data struct { + Version string `json:"version"` + } `json:"data"` + } + + if err = json.Unmarshal(body, &b); err != nil { + return "", errors.Wrap(err, "unmarshal build info API response") + } + + return b.Data.Version, nil +} + func formatTime(t time.Time) string { return strconv.FormatFloat(float64(t.Unix())+float64(t.Nanosecond())/1e9, 'f', -1, 64) } diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index ce2b905db8..82fd7eda59 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -50,6 +50,7 @@ type PrometheusStore struct { component component.StoreAPI externalLabelsFn func() labels.Labels timestamps func() (mint int64, maxt int64) + promVersion func() string remoteReadAcceptableResponses []prompb.ReadRequest_ResponseType @@ -69,6 +70,7 @@ func NewPrometheusStore( component component.StoreAPI, externalLabelsFn func() labels.Labels, timestamps func() (mint int64, maxt int64), + promVersion func() string, ) (*PrometheusStore, error) { if logger == nil { logger = log.NewNopLogger() @@ -80,6 +82,7 @@ func NewPrometheusStore( component: component, externalLabelsFn: externalLabelsFn, timestamps: timestamps, + promVersion: promVersion, remoteReadAcceptableResponses: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS, prompb.ReadRequest_SAMPLES}, buffers: sync.Pool{New: func() interface{} { b := make([]byte, 0, initialBufSize) @@ -496,10 +499,31 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return &storepb.LabelValuesResponse{Values: []string{l}}, nil } - vals, err := p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) - if err != nil { - return nil, err + var ( + vals []string + sers []map[string]string + err error + ) + + version := p.promVersion() + if version > "2.24" { + vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) + if err != nil { + return nil, err + } + sort.Strings(vals) + } else { + matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + if err != nil { + return nil, err + } + for _, s := range sers { + vals = append(vals, s[r.Label]) + } } - sort.Strings(vals) - return &storepb.LabelValuesResponse{Values: vals}, nil + return &storepb.LabelValuesResponse{Values: vals}, nil } diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 38cca22e5f..f963a4ed9a 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -64,7 +64,7 @@ func testPrometheusStoreSeriesE2e(t *testing.T, prefix string) { limitMinT := int64(0) proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return limitMinT, -1 }) // Maxt does not matter. + func() (int64, int64) { return limitMinT, -1 }, nil) // Maxt does not matter. testutil.Ok(t, err) // Query all three samples except for the first one. Since we round up queried data @@ -199,7 +199,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { promStore, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return math.MinInt64/1000 + 62135596801, math.MaxInt64/1000 - 62135596801 }) + func() (int64, int64) { return math.MinInt64/1000 + 62135596801, math.MaxInt64/1000 - 62135596801 }, nil) testutil.Ok(t, err) for _, tcase := range []struct { @@ -375,7 +375,7 @@ func TestPrometheusStore_LabelNames_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil) + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, nil) testutil.Ok(t, err) resp, err := proxy.LabelNames(ctx, &storepb.LabelNamesRequest{ @@ -420,7 +420,8 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil) + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, + func() string {return "2.25"}) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ @@ -492,7 +493,7 @@ func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil) + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, nil) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ @@ -538,7 +539,7 @@ func TestPrometheusStore_Series_MatchExternalLabel_e2e(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }) + func() (int64, int64) { return 0, math.MaxInt64 }, nil) testutil.Ok(t, err) srv := newStoreSeriesServer(ctx) @@ -583,7 +584,7 @@ func TestPrometheusStore_Info(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), nil, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 123, 456 }) + func() (int64, int64) { return 123, 456 }, nil) testutil.Ok(t, err) resp, err := proxy.Info(ctx, &storepb.InfoRequest{}) @@ -661,7 +662,7 @@ func TestPrometheusStore_Series_SplitSamplesIntoChunksWithMaxSizeOf120(t *testin proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }) + func() (int64, int64) { return 0, math.MaxInt64 }, nil) testutil.Ok(t, err) // We build chunks only for SAMPLES method. Make sure we ask for SAMPLES only. From f4096da200bdffad4d493bf238f700fdc9ac6887 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Fri, 19 Mar 2021 01:54:46 +0530 Subject: [PATCH 03/15] go formatting fix Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 8 ++++---- pkg/store/prometheus.go | 12 ++++++------ pkg/store/prometheus_test.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 8a8cd3057f..0d67cc89eb 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -371,10 +371,10 @@ func validatePrometheus(ctx context.Context, client *promclient.Client, logger l type promMetadata struct { promURL *url.URL - mtx sync.Mutex - mint int64 - maxt int64 - labels labels.Labels + mtx sync.Mutex + mint int64 + maxt int64 + labels labels.Labels promVersion string limitMinTime thanosmodel.TimeOrDurationValue diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 82fd7eda59..fbcff7037b 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -50,7 +50,7 @@ type PrometheusStore struct { component component.StoreAPI externalLabelsFn func() labels.Labels timestamps func() (mint int64, maxt int64) - promVersion func() string + promVersion func() string remoteReadAcceptableResponses []prompb.ReadRequest_ResponseType @@ -82,7 +82,7 @@ func NewPrometheusStore( component: component, externalLabelsFn: externalLabelsFn, timestamps: timestamps, - promVersion: promVersion, + promVersion: promVersion, remoteReadAcceptableResponses: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS, prompb.ReadRequest_SAMPLES}, buffers: sync.Pool{New: func() interface{} { b := make([]byte, 0, initialBufSize) @@ -500,9 +500,9 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue } var ( - vals []string - sers []map[string]string - err error + vals []string + sers []map[string]string + err error ) version := p.promVersion() @@ -525,5 +525,5 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue vals = append(vals, s[r.Label]) } } - return &storepb.LabelValuesResponse{Values: vals}, nil + return &storepb.LabelValuesResponse{Values: vals}, nil } diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index f963a4ed9a..ce32b32c60 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -420,8 +420,8 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, - func() string {return "2.25"}) + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, + func() string { return "2.25" }) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ From 6c2ec330a28cb67a51d992c242440e4b614936fa Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Sat, 27 Mar 2021 23:39:54 +0530 Subject: [PATCH 04/15] addressed minor changes Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 4 ++-- pkg/promclient/promclient.go | 8 ++++---- pkg/store/prometheus.go | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 7ac3dcc04f..e58e6779dd 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -212,7 +212,7 @@ func runSidecar( { ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { - // We retry infinitely until we reach and fetch BuildInfo from our Prometheus. + // We retry infinitely until we reach and fetch BuildVersion from our Prometheus. err := runutil.Retry(2*time.Second, ctx.Done(), func() error { if err := m.BuildVersion(ctx); err != nil { level.Warn(logger).Log( @@ -424,7 +424,7 @@ func (s *promMetadata) Timestamps() (mint int64, maxt int64) { } func (s *promMetadata) BuildVersion(ctx context.Context) error { - ver, err := s.client.BuildInfo(ctx, s.promURL) + ver, err := s.client.BuildVersion(ctx, s.promURL) if err != nil { return err } diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 6ea356b94d..c99cc49353 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -608,14 +608,14 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode return v.Data, nil } -// BuildInfo returns Prometheus version from /api/v1/status/buildinfo Prometheus endpoint. -func (c *Client) BuildInfo(ctx context.Context, base *url.URL) (string, error) { +// BuildVersion returns Prometheus version from /api/v1/status/buildinfo Prometheus endpoint. +func (c *Client) BuildVersion(ctx context.Context, base *url.URL) (string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/status/buildinfo") - level.Debug(c.logger).Log("msg", "querying instant", "url", u.String()) + level.Debug(c.logger).Log("msg", "build version", "url", u.String()) - span, ctx := tracing.StartSpan(ctx, "/prom_buildinfo HTTP[client]") + span, ctx := tracing.StartSpan(ctx, "/prom_buildversion HTTP[client]") defer span.Finish() body, _, err := c.req2xx(ctx, &u, http.MethodGet) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index fbcff7037b..37f7b369dd 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -511,7 +511,6 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue if err != nil { return nil, err } - sort.Strings(vals) } else { matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) if err != nil { @@ -522,8 +521,11 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return nil, err } for _, s := range sers { - vals = append(vals, s[r.Label]) + if val, exists := s[r.Label]; exists { + vals = append(vals, val) + } } } + sort.Strings(vals) return &storepb.LabelValuesResponse{Values: vals}, nil } From a4073e4ba44d460a5a48858f39d301b376ab91ee Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Sat, 3 Apr 2021 23:36:54 +0530 Subject: [PATCH 05/15] using set to handle duplicate values Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 2 +- pkg/store/prometheus.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index e58e6779dd..fd00fbe177 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -229,7 +229,7 @@ func runSidecar( return nil }) if err != nil { - return errors.Wrap(err, "buildinfo query") + return errors.Wrap(err, "failed to get prometheus version") } return nil }, func(error) { diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 37f7b369dd..b1bf61e65e 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -520,11 +520,17 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue if err != nil { return nil, err } + + // using set to handle duplicate values. + labelValuesSet := make(map[string]struct{}) for _, s := range sers { if val, exists := s[r.Label]; exists { - vals = append(vals, val) + labelValuesSet[val] = struct{}{} } } + for key := range labelValuesSet { + vals = append(vals, key) + } } sort.Strings(vals) return &storepb.LabelValuesResponse{Values: vals}, nil From fe5067a7eb1de04a2990ca0e86d23ca60c36f88b Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Wed, 14 Apr 2021 17:28:54 +0530 Subject: [PATCH 06/15] Logic correction Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 42 +++++++++++++++++++++--------------- pkg/store/prometheus_test.go | 5 ++++- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index b1bf61e65e..8d93499c5f 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -500,36 +500,44 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue } var ( - vals []string sers []map[string]string err error ) + vals := []string{} version := p.promVersion() + if version > "2.24" { vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) if err != nil { return nil, err } } else { - matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) - if err != nil { - return nil, err - } + if len(r.Matchers) == 0 { + vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, nil, r.Start, r.End) + if err != nil { + return nil, err + } + } else { + matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + if err != nil { + return nil, err + } - // using set to handle duplicate values. - labelValuesSet := make(map[string]struct{}) - for _, s := range sers { - if val, exists := s[r.Label]; exists { - labelValuesSet[val] = struct{}{} + // using set to handle duplicate values. + labelValuesSet := make(map[string]struct{}) + for _, s := range sers { + if val, exists := s[r.Label]; exists { + labelValuesSet[val] = struct{}{} + } + } + for key := range labelValuesSet { + vals = append(vals, key) } - } - for key := range labelValuesSet { - vals = append(vals, key) } } sort.Strings(vals) diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 94ae03436a..01d4c2240e 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -420,8 +420,11 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) + version, err := promclient.NewDefaultClient().BuildVersion(ctx, u) + testutil.Ok(t, err) + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, - func() string { return "2.25" }) + func() string { return version }) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ From 92c3234ca4bc9b824f476a6d3b7f5ac97a135e70 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Wed, 14 Apr 2021 18:00:13 +0530 Subject: [PATCH 07/15] test case update Signed-off-by: Namanl2001 --- pkg/store/prometheus_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 01d4c2240e..c99d0375f5 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -496,7 +496,10 @@ func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, nil) + version, err := promclient.NewDefaultClient().BuildVersion(ctx, u) + testutil.Ok(t, err) + + proxy, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, getExternalLabels, nil, func() string { return version }) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ From df6b6efa7680c65cd6f8715ae43357aff99eb528 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Wed, 14 Apr 2021 19:56:37 +0530 Subject: [PATCH 08/15] changes addressed Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 8d93499c5f..b828424940 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -507,38 +507,31 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue vals := []string{} version := p.promVersion() - if version > "2.24" { + if len(r.Matchers) == 0 || version > "2.24" { vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) if err != nil { return nil, err } } else { - if len(r.Matchers) == 0 { - vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, nil, r.Start, r.End) - if err != nil { - return nil, err - } - } else { - matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) - if err != nil { - return nil, err - } + matchers, err := storepb.MatchersToPromMatchers(r.Matchers...) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + if err != nil { + return nil, err + } - // using set to handle duplicate values. - labelValuesSet := make(map[string]struct{}) - for _, s := range sers { - if val, exists := s[r.Label]; exists { - labelValuesSet[val] = struct{}{} - } - } - for key := range labelValuesSet { - vals = append(vals, key) + // using set to handle duplicate values. + labelValuesSet := make(map[string]struct{}) + for _, s := range sers { + if val, exists := s[r.Label]; exists { + labelValuesSet[val] = struct{}{} } } + for key := range labelValuesSet { + vals = append(vals, key) + } } sort.Strings(vals) return &storepb.LabelValuesResponse{Values: vals}, nil From aaf7906a83af5810d7d3f42d9ce490c8ad8f5729 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Sat, 17 Apr 2021 00:01:14 +0530 Subject: [PATCH 09/15] handling promVersion < 2.14 Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 3 +-- pkg/promclient/promclient.go | 7 ++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index fd00fbe177..156666fd95 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -216,7 +216,7 @@ func runSidecar( err := runutil.Retry(2*time.Second, ctx.Done(), func() error { if err := m.BuildVersion(ctx); err != nil { level.Warn(logger).Log( - "msg", "failed to fetch prometheur version. Is Prometheus running? Retrying", + "msg", "failed to fetch prometheus version. Is Prometheus running? Retrying", "err", err, ) return err @@ -224,7 +224,6 @@ func runSidecar( level.Info(logger).Log( "msg", "successfully loaded prometheus version", - "prometheus_version", m.Version(), ) return nil }) diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index c99cc49353..4384b5d529 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -609,6 +609,7 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode } // BuildVersion returns Prometheus version from /api/v1/status/buildinfo Prometheus endpoint. +// For Prometheus versions < 2.14.0 it returns "0" as Prometheus version. func (c *Client) BuildVersion(ctx context.Context, base *url.URL) (string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/status/buildinfo") @@ -618,8 +619,12 @@ func (c *Client) BuildVersion(ctx context.Context, base *url.URL) (string, error span, ctx := tracing.StartSpan(ctx, "/prom_buildversion HTTP[client]") defer span.Finish() - body, _, err := c.req2xx(ctx, &u, http.MethodGet) + // We get status code 404 for prometheus versions lower than 2.14.0 + body, code, err := c.req2xx(ctx, &u, http.MethodGet) if err != nil { + if code == http.StatusNotFound { + return "0", nil + } return "", err } From ef797f682c7c53a369ca4a06aefc06e2ea0bae3d Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Sat, 24 Apr 2021 02:50:07 +0530 Subject: [PATCH 10/15] removed goroutine for promVersion Signed-off-by: Namanl2001 --- cmd/thanos/sidecar.go | 50 +++++++++++++++++------------------------ pkg/store/prometheus.go | 2 +- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 156666fd95..790ddc230c 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -145,15 +145,34 @@ func runSidecar( g.Add(func() error { // Only check Prometheus's flags when upload is enabled. if uploads { - // Check prometheus's flags to ensure sane sidecar flags. + // Check prometheus's flags to ensure same sidecar flags. if err := validatePrometheus(ctx, m.client, logger, conf.shipper.ignoreBlockSize, m); err != nil { return errors.Wrap(err, "validate Prometheus flags") } } + // We retry infinitely until we reach and fetch BuildVersion from our Prometheus. + err := runutil.Retry(2*time.Second, ctx.Done(), func() error { + if err := m.BuildVersion(ctx); err != nil { + level.Warn(logger).Log( + "msg", "failed to fetch prometheus version. Is Prometheus running? Retrying", + "err", err, + ) + return err + } + + level.Info(logger).Log( + "msg", "successfully loaded prometheus version", + ) + return nil + }) + if err != nil { + return errors.Wrap(err, "failed to get prometheus version") + } + // Blocking query of external labels before joining as a Source Peer into gossip. // We retry infinitely until we reach and fetch labels from our Prometheus. - err := runutil.Retry(2*time.Second, ctx.Done(), func() error { + err = runutil.Retry(2*time.Second, ctx.Done(), func() error { if err := m.UpdateLabels(ctx); err != nil { level.Warn(logger).Log( "msg", "failed to fetch initial external labels. Is Prometheus running? Retrying", @@ -209,33 +228,6 @@ func runSidecar( cancel() }) } - { - ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { - // We retry infinitely until we reach and fetch BuildVersion from our Prometheus. - err := runutil.Retry(2*time.Second, ctx.Done(), func() error { - if err := m.BuildVersion(ctx); err != nil { - level.Warn(logger).Log( - "msg", "failed to fetch prometheus version. Is Prometheus running? Retrying", - "err", err, - ) - return err - } - - level.Info(logger).Log( - "msg", "successfully loaded prometheus version", - ) - return nil - }) - if err != nil { - return errors.Wrap(err, "failed to get prometheus version") - } - return nil - }, func(error) { - cancel() - }) - } - { t := exthttp.NewTransport() t.MaxIdleConnsPerHost = conf.connection.maxIdleConnsPerHost diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index b828424940..abbc7cfee7 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -522,7 +522,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return nil, err } - // using set to handle duplicate values. + // Using set to handle duplicate values. labelValuesSet := make(map[string]struct{}) for _, s := range sers { if val, exists := s[r.Label]; exists { From fd708fe04877c8d4e404490a0e37130e27fcb89e Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Wed, 28 Apr 2021 00:11:21 +0530 Subject: [PATCH 11/15] using semver Signed-off-by: Namanl2001 --- go.mod | 1 + go.sum | 4 ++++ pkg/store/prometheus.go | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9f349b14f2..116899c53c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible + github.com/blang/semver/v4 v4.0.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/cespare/xxhash v1.1.0 github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0 diff --git a/go.sum b/go.sum index d070c4160e..5adfef6f78 100644 --- a/go.sum +++ b/go.sum @@ -181,7 +181,11 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index abbc7cfee7..3ea7870cfe 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -17,6 +17,7 @@ import ( "strings" "sync" + "github.com/blang/semver/v4" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gogo/protobuf/proto" @@ -504,10 +505,22 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue err error ) + lvc := false // LabelValuesCall vals := []string{} - version := p.promVersion() + v := p.promVersion() - if len(r.Matchers) == 0 || version > "2.24" { + version, err := semver.Parse(v) + if err == nil { + baseVer, err := semver.Make("2.24.0") + if err != nil { + return nil, err + } + if version.Compare(baseVer) == 1 { + lvc = true + } + } + + if len(r.Matchers) == 0 || lvc { vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Matchers, r.Start, r.End) if err != nil { return nil, err From d874bc132a5006b6b5700df2c6f2948c7a137eac Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Wed, 28 Apr 2021 00:28:05 +0530 Subject: [PATCH 12/15] making linter happy1 Signed-off-by: Namanl2001 --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 5adfef6f78..6561def31a 100644 --- a/go.sum +++ b/go.sum @@ -183,7 +183,6 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCS github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= From e59208c9162100c80cb02f5ccea7b4bea9d54915 Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Thu, 29 Apr 2021 19:43:53 +0530 Subject: [PATCH 13/15] small nits Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 3ea7870cfe..e726ba1be5 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -509,15 +509,11 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue vals := []string{} v := p.promVersion() - version, err := semver.Parse(v) - if err == nil { - baseVer, err := semver.Make("2.24.0") - if err != nil { - return nil, err - } - if version.Compare(baseVer) == 1 { - lvc = true - } + version, err1 := semver.Parse(v) + baseVer, err2 := semver.Make("2.24.0") + + if err1 == nil && err2 == nil && version.Compare(baseVer) == 1 { + lvc = true } if len(r.Matchers) == 0 || lvc { From 893e37cf7118a4ab6e9388810a5877c87afbef4d Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Fri, 30 Apr 2021 02:12:37 +0530 Subject: [PATCH 14/15] made baseVer global Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index e726ba1be5..562335a157 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -58,6 +58,9 @@ type PrometheusStore struct { framesRead prometheus.Histogram } +// LabelValues call with matchers is supported for Prometheus versions >= 2.24.0 +var baseVer, _ = semver.Make("2.24.0") + const initialBufSize = 32 * 1024 // 32KB seems like a good minimum starting size for sync pool size. // NewPrometheusStore returns a new PrometheusStore that uses the given HTTP client @@ -509,10 +512,8 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue vals := []string{} v := p.promVersion() - version, err1 := semver.Parse(v) - baseVer, err2 := semver.Make("2.24.0") - - if err1 == nil && err2 == nil && version.Compare(baseVer) == 1 { + version, err := semver.Parse(v) + if err == nil && version.GTE(baseVer) { lvc = true } From 92570d0b3cca389aa9b269b27247b6659144afea Mon Sep 17 00:00:00 2001 From: Namanl2001 Date: Fri, 30 Apr 2021 02:23:31 +0530 Subject: [PATCH 15/15] linting correction Signed-off-by: Namanl2001 --- pkg/store/prometheus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 562335a157..894cc34893 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -58,7 +58,7 @@ type PrometheusStore struct { framesRead prometheus.Histogram } -// LabelValues call with matchers is supported for Prometheus versions >= 2.24.0 +// LabelValues call with matchers is supported for Prometheus versions >= 2.24.0 . var baseVer, _ = semver.Make("2.24.0") const initialBufSize = 32 * 1024 // 32KB seems like a good minimum starting size for sync pool size.