diff --git a/CHANGELOG.md b/CHANGELOG.md index bb976721d29..baf21326314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7609](https://github.com/thanos-io/thanos/pull/7609) API: Add limit param to metadata APIs (series, label names, label values). - [#7429](https://github.com/thanos-io/thanos/pull/7429): Reloader: introduce `TolerateEnvVarExpansionErrors` to allow suppressing errors when expanding environment variables in the configuration file. When set, this will ensure that the reloader won't consider the operation to fail when an unset environment variable is encountered. Note that all unset environment variables are left as is, whereas all set environment variables are expanded as usual. - [#7560](https://github.com/thanos-io/thanos/pull/7560) Query: Added the possibility of filtering rules by rule_name, rule_group or file to HTTP api. +- [#7652](https://github.com/thanos-io/thanos/pull/7652) Store: Implement metadata API limit in stores. ### Changed diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 5dde62c5eec..1f96f4c666f 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -734,7 +734,7 @@ func (c *Client) get2xxResultWithGRPCErrors(ctx context.Context, spanName string // SeriesInGRPC returns the labels from Prometheus series API. It uses gRPC errors. // NOTE: This method is tested in pkg/store/prometheus_test.go against Prometheus. -func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []*labels.Matcher, startTime, endTime int64) ([]map[string]string, error) { +func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []*labels.Matcher, startTime, endTime int64, limit int) ([]map[string]string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/series") q := u.Query() @@ -742,6 +742,7 @@ func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []*la q.Add("match[]", storepb.PromMatchersToString(matchers...)) q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) + q.Add("limit", strconv.Itoa(limit)) u.RawQuery = q.Encode() var m struct { @@ -753,7 +754,7 @@ func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []*la // LabelNamesInGRPC returns all known label names constrained by the given matchers. It uses gRPC errors. // NOTE: This method is tested in pkg/store/prometheus_test.go against Prometheus. -func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers []*labels.Matcher, startTime, endTime int64) ([]string, error) { +func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers []*labels.Matcher, startTime, endTime int64, limit int) ([]string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/labels") q := u.Query() @@ -763,6 +764,7 @@ func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers [ } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) + q.Add("limit", strconv.Itoa(limit)) u.RawQuery = q.Encode() var m struct { @@ -773,7 +775,7 @@ func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers [ // LabelValuesInGRPC returns all known label values for a given label name. It uses gRPC errors. // NOTE: This method is tested in pkg/store/prometheus_test.go against Prometheus. -func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label string, matchers []*labels.Matcher, startTime, endTime int64) ([]string, error) { +func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label string, matchers []*labels.Matcher, startTime, endTime int64, limit int) ([]string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/label/", label, "/values") q := u.Query() @@ -783,6 +785,7 @@ func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label str } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) + q.Add("limit", strconv.Itoa(limit)) u.RawQuery = q.Encode() var m struct { diff --git a/pkg/query/querier.go b/pkg/query/querier.go index 9fddae11a5a..e084344ed91 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -331,6 +331,7 @@ func (q *querier) selectFn(ctx context.Context, hints *storage.SelectHints, ms . req := storepb.SeriesRequest{ MinTime: hints.Start, MaxTime: hints.End, + Limit: int64(hints.Limit), Matchers: sms, MaxResolutionWindow: q.maxResolutionMillis, Aggregates: aggrs, @@ -373,7 +374,7 @@ func (q *querier) selectFn(ctx context.Context, hints *storage.SelectHints, ms . } // LabelValues returns all potential values for a label name. -func (q *querier) LabelValues(ctx context.Context, name string, _ *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { +func (q *querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { span, ctx := tracing.StartSpan(ctx, "querier_label_values") defer span.Finish() @@ -384,12 +385,18 @@ func (q *querier) LabelValues(ctx context.Context, name string, _ *storage.Label if err != nil { return nil, nil, errors.Wrap(err, "converting prom matchers to storepb matchers") } + + if hints == nil { + hints = &storage.LabelHints{} + } + req := &storepb.LabelValuesRequest{ Label: name, PartialResponseStrategy: q.partialResponseStrategy, Start: q.mint, End: q.maxt, Matchers: pbMatchers, + Limit: int64(hints.Limit), } if q.isDedupEnabled() { @@ -411,7 +418,7 @@ func (q *querier) LabelValues(ctx context.Context, name string, _ *storage.Label // LabelNames returns all the unique label names present in the block in sorted order constrained // by the given matchers. -func (q *querier) LabelNames(ctx context.Context, _ *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { +func (q *querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { span, ctx := tracing.StartSpan(ctx, "querier_label_names") defer span.Finish() @@ -423,11 +430,16 @@ func (q *querier) LabelNames(ctx context.Context, _ *storage.LabelHints, matcher return nil, nil, errors.Wrap(err, "converting prom matchers to storepb matchers") } + if hints == nil { + hints = &storage.LabelHints{} + } + req := &storepb.LabelNamesRequest{ PartialResponseStrategy: q.partialResponseStrategy, Start: q.mint, End: q.maxt, Matchers: pbMatchers, + Limit: int64(hints.Limit), } if q.isDedupEnabled() { diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 957910d7f41..43ae5193113 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1006,11 +1006,12 @@ type blockSeriesClient struct { extLset labels.Labels extLsetToRemove map[string]struct{} - mint int64 - maxt int64 - indexr *bucketIndexReader - chunkr *bucketChunkReader - loadAggregates []storepb.Aggr + mint int64 + maxt int64 + expandedPostingsLimit int + indexr *bucketIndexReader + chunkr *bucketChunkReader + loadAggregates []storepb.Aggr seriesLimiter SeriesLimiter chunksLimiter ChunksLimiter @@ -1083,6 +1084,7 @@ func newBlockSeriesClient( mint: req.MinTime, maxt: req.MaxTime, + expandedPostingsLimit: int(req.Limit), indexr: b.indexReader(logger), chunkr: chunkr, seriesLimiter: seriesLimiter, @@ -1162,14 +1164,20 @@ func (b *blockSeriesClient) ExpandPostings( b.expandedPostings = make([]storage.SeriesRef, 0, len(b.lazyPostings.postings)/2) b.lazyExpandedPostingsCount.Inc() } else { + // If expandedPostingsLimit is set, it can be applied here to limit the amount of series. + // Note: This can only be done when postings are not expanded lazily. + if b.expandedPostingsLimit > 0 && len(b.lazyPostings.postings) > b.expandedPostingsLimit { + b.lazyPostings.postings = b.lazyPostings.postings[:b.expandedPostingsLimit] + } + // Apply series limiter eargerly if lazy postings not enabled. - if err := seriesLimiter.Reserve(uint64(len(ps.postings))); err != nil { + if err := seriesLimiter.Reserve(uint64(len(b.lazyPostings.postings))); err != nil { return httpgrpc.Errorf(int(codes.ResourceExhausted), "exceeded series limit: %s", err) } } - if b.batchSize > len(ps.postings) { - b.batchSize = len(ps.postings) + if b.batchSize > len(b.lazyPostings.postings) { + b.batchSize = len(b.lazyPostings.postings) } b.entries = make([]seriesEntry, 0, b.batchSize) @@ -1694,7 +1702,12 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, seriesSrv storepb.Store tracing.DoInSpan(ctx, "bucket_store_merge_all", func(ctx context.Context) { begin := time.Now() set := NewResponseDeduplicator(NewProxyResponseLoserTree(respSets...)) + i := 0 for set.Next() { + i++ + if req.Limit > 0 && i > int(req.Limit) { + break + } at := set.At() warn := at.GetWarning() if warn != "" { @@ -1945,8 +1958,13 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq return nil, status.Error(codes.Unknown, errors.Wrap(err, "marshal label names response hints").Error()) } + names := strutil.MergeSlices(sets...) + if req.Limit > 0 && len(names) > int(req.Limit) { + names = names[:req.Limit] + } + return &storepb.LabelNamesResponse{ - Names: strutil.MergeSlices(sets...), + Names: names, Hints: anyHints, }, nil } @@ -2160,8 +2178,13 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR return nil, status.Error(codes.Unknown, errors.Wrap(err, "marshal label values response hints").Error()) } + vals := strutil.MergeSlices(sets...) + if req.Limit > 0 && len(vals) > int(req.Limit) { + vals = vals[:req.Limit] + } + return &storepb.LabelValuesResponse{ - Values: strutil.MergeSlices(sets...), + Values: vals, Hints: anyHints, }, nil } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 7a26d14b528..7b20b44772e 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -3935,3 +3935,138 @@ func (m *compositeBytesLimiterMock) ReserveWithType(num uint64, dataType StoreDa } return nil } + +func TestBucketStoreMetadataLimit(t *testing.T) { + tb := testutil.NewTB(t) + + tmpDir := t.TempDir() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(tb, err) + defer func() { testutil.Ok(tb, bkt.Close()) }() + + uploadTestBlock(tb, tmpDir, bkt, 100) + + instrBkt := objstore.WithNoopInstr(bkt) + logger := log.NewNopLogger() + + // Instance a real bucket store we'll use to query the series. + baseBlockIDsFetcher := block.NewConcurrentLister(logger, instrBkt) + fetcher, err := block.NewMetaFetcher(logger, 10, instrBkt, baseBlockIDsFetcher, tmpDir, nil, nil) + testutil.Ok(tb, err) + + indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(logger, nil, nil, storecache.InMemoryIndexCacheConfig{}) + testutil.Ok(tb, err) + + store, err := NewBucketStore( + instrBkt, + fetcher, + tmpDir, + NewChunksLimiterFactory(0), + NewSeriesLimiterFactory(0), + NewBytesLimiterFactory(0), + NewGapBasedPartitioner(PartitionerMaxGapSize), + 10, + false, + DefaultPostingOffsetInMemorySampling, + true, + false, + 0, + WithLogger(logger), + WithIndexCache(indexCache), + ) + testutil.Ok(tb, err) + testutil.Ok(tb, store.SyncBlocks(context.Background())) + + seriesTests := map[string]struct { + limit int64 + expectedResults int + }{ + "series without limit": { + expectedResults: 40, + }, + "series with limit": { + limit: 2, + expectedResults: 2, + }, + } + + for testName, testData := range seriesTests { + t.Run(testName, func(t *testing.T) { + req := &storepb.SeriesRequest{ + MinTime: timestamp.FromTime(minTime), + MaxTime: timestamp.FromTime(maxTime), + Limit: testData.limit, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "j", Value: "foo"}, + }, + } + + srv := newStoreSeriesServer(context.Background()) + err = store.Series(req, srv) + testutil.Ok(t, err) + testutil.Assert(t, len(srv.SeriesSet) == testData.expectedResults) + }) + } + + labelNamesTests := map[string]struct { + limit int64 + expectedResults []string + }{ + "label names without limit": { + expectedResults: []string{"ext1", "i", "j", "n", "uniq"}, + }, + "label names with limit": { + limit: 3, + expectedResults: []string{"ext1", "i", "j"}, + }, + } + + for testName, testData := range labelNamesTests { + t.Run(testName, func(t *testing.T) { + req := &storepb.LabelNamesRequest{ + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + Limit: testData.limit, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "j", Value: "foo"}, + }, + } + + resp, err := store.LabelNames(context.Background(), req) + testutil.Ok(t, err) + testutil.Equals(t, testData.expectedResults, resp.Names) + }) + } + + labelValuesTests := map[string]struct { + limit int64 + expectedResults []string + }{ + "label values without limit": { + expectedResults: []string{"bar", "foo"}, + }, + "label values with limit": { + limit: 1, + expectedResults: []string{"bar"}, + }, + } + + for testName, testData := range labelValuesTests { + t.Run(testName, func(t *testing.T) { + req := &storepb.LabelValuesRequest{ + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + Label: "j", + Limit: testData.limit, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_RE, Name: "j", Value: "(foo|bar)"}, + }, + } + + resp, err := store.LabelValues(context.Background(), req) + testutil.Ok(t, err) + testutil.Equals(t, testData.expectedResults, resp.Values) + }) + } +} diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 721e9ed51ee..29462789781 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -173,7 +173,7 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, seriesSrv storepb.Sto if r.SkipChunks { finalExtLset := rmLabels(extLset.Copy(), extLsetToRemove) - labelMaps, err := p.client.SeriesInGRPC(s.Context(), p.base, matchers, r.MinTime, r.MaxTime) + labelMaps, err := p.client.SeriesInGRPC(s.Context(), p.base, matchers, r.MinTime, r.MaxTime, int(r.Limit)) if err != nil { return err } @@ -571,12 +571,12 @@ func (p *PrometheusStore) LabelNames(ctx context.Context, r *storepb.LabelNamesR var lbls []string if len(matchers) == 0 || p.labelCallsSupportMatchers() { - lbls, err = p.client.LabelNamesInGRPC(ctx, p.base, matchers, r.Start, r.End) + lbls, err = p.client.LabelNamesInGRPC(ctx, p.base, matchers, r.Start, r.End, int(r.Limit)) if err != nil { return nil, err } } else { - sers, err := p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + sers, err := p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End, int(r.Limit)) if err != nil { return nil, err } @@ -642,7 +642,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue if len(matchers) == 0 { return &storepb.LabelValuesResponse{Values: []string{val}}, nil } - sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End, int(r.Limit)) if err != nil { return nil, err } @@ -653,12 +653,12 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue } if len(matchers) == 0 || p.labelCallsSupportMatchers() { - vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, matchers, r.Start, r.End) + vals, err = p.client.LabelValuesInGRPC(ctx, p.base, r.Label, matchers, r.Start, r.End, int(r.Limit)) if err != nil { return nil, err } } else { - sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End) + sers, err = p.client.SeriesInGRPC(ctx, p.base, matchers, r.Start, r.End, int(r.Limit)) if err != nil { return nil, err } diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 0ac1fc659cc..3d21a8631fa 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -327,6 +327,7 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb. r := &storepb.SeriesRequest{ MinTime: originalRequest.MinTime, MaxTime: originalRequest.MaxTime, + Limit: originalRequest.Limit, Matchers: append(storeMatchers, MatchersForLabelSets(storeLabelSets)...), Aggregates: originalRequest.Aggregates, MaxResolutionWindow: originalRequest.MaxResolutionWindow, @@ -363,7 +364,13 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb. level.Debug(reqLogger).Log("msg", "Series: started fanout streams", "status", strings.Join(storeDebugMsgs, ";")) respHeap := NewResponseDeduplicator(NewProxyResponseLoserTree(storeResponses...)) + + i := 0 for respHeap.Next() { + i++ + if r.Limit > 0 && i > int(r.Limit) { + break + } resp := respHeap.At() if resp.GetWarning() != "" && (r.PartialResponseDisabled || r.PartialResponseStrategy == storepb.PartialResponseStrategy_ABORT) { @@ -419,6 +426,7 @@ func (s *ProxyStore) LabelNames(ctx context.Context, originalRequest *storepb.La End: originalRequest.End, Matchers: append(storeMatchers, MatchersForLabelSets(storeLabelSets)...), WithoutReplicaLabels: originalRequest.WithoutReplicaLabels, + Hints: originalRequest.Hints, } var ( @@ -465,8 +473,13 @@ func (s *ProxyStore) LabelNames(ctx context.Context, originalRequest *storepb.La return nil, err } + result := strutil.MergeUnsortedSlices(names...) + if originalRequest.Limit > 0 && len(result) > int(originalRequest.Limit) { + result = result[:originalRequest.Limit] + } + return &storepb.LabelNamesResponse{ - Names: strutil.MergeUnsortedSlices(names...), + Names: result, Warnings: warnings, }, nil } @@ -520,6 +533,7 @@ func (s *ProxyStore) LabelValues(ctx context.Context, originalRequest *storepb.L End: originalRequest.End, Matchers: append(storeMatchers, MatchersForLabelSets(storeLabelSets)...), WithoutReplicaLabels: originalRequest.WithoutReplicaLabels, + Limit: originalRequest.Limit, } var ( @@ -567,8 +581,13 @@ func (s *ProxyStore) LabelValues(ctx context.Context, originalRequest *storepb.L return nil, err } + vals := strutil.MergeUnsortedSlices(all...) + if originalRequest.Limit > 0 && len(vals) > int(originalRequest.Limit) { + vals = vals[:originalRequest.Limit] + } + return &storepb.LabelValuesResponse{ - Values: strutil.MergeUnsortedSlices(all...), + Values: vals, Warnings: warnings, }, nil } diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index 137383fa57a..d39e02e5f04 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -742,6 +742,46 @@ func TestProxyStore_Series(t *testing.T) { }, }, }, + { + title: "skip chunks with limit", + storeAPIs: []Client{ + &storetestutil.TestClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "1")), + storeSeriesResponse(t, labels.FromStrings("a", "2")), + }, + }, + MinTime: 1, + MaxTime: 300, + }, + &storetestutil.TestClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "3")), + storeSeriesResponse(t, labels.FromStrings("a", "4")), + }, + }, + MinTime: 1, + MaxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "a", Value: "[1-4]", Type: storepb.LabelMatcher_RE}}, + SkipChunks: true, + Limit: 2, + }, + expectedSeries: []rawSeries{ + { + lset: labels.FromStrings("a", "1"), + }, + { + lset: labels.FromStrings("a", "2"), + }, + }, + }, } { t.Run(tc.title, func(t *testing.T) { for _, replicaLabelSupport := range []bool{false, true} { @@ -1439,7 +1479,7 @@ func TestProxyStore_LabelValues(t *testing.T) { Warnings: []string{"warning"}, }, } - cls := []Client{ + storeApis := []Client{ &storetestutil.TestClient{StoreClient: m1}, &storetestutil.TestClient{StoreClient: &mockedStoreAPI{ RespLabelValues: &storepb.LabelValuesResponse{ @@ -1454,41 +1494,82 @@ func TestProxyStore_LabelValues(t *testing.T) { MaxTime: timestamp.FromTime(time.Now()), }, } - q := NewProxyStore(nil, - nil, - func() []Client { return cls }, - component.Query, - labels.EmptyLabels(), - 0*time.Second, EagerRetrieval, - ) - ctx := context.Background() - req := &storepb.LabelValuesRequest{ - Label: "a", - PartialResponseDisabled: true, - Start: timestamp.FromTime(minTime), - End: timestamp.FromTime(maxTime), - } - resp, err := q.LabelValues(ctx, req) - testutil.Ok(t, err) - testutil.Assert(t, proto.Equal(req, m1.LastLabelValuesReq), "request was not proxied properly to underlying storeAPI: %s vs %s", req, m1.LastLabelValuesReq) + for _, tc := range []struct { + title string + storeAPIs []Client - testutil.Equals(t, []string{"1", "2", "3", "4", "5", "6"}, resp.Values) - testutil.Equals(t, 1, len(resp.Warnings)) + req *storepb.LabelValuesRequest - // Request outside the time range of the last store client. - req = &storepb.LabelValuesRequest{ - Label: "a", - PartialResponseDisabled: true, - Start: timestamp.FromTime(minTime), - End: timestamp.FromTime(time.Now().Add(-1 * time.Hour)), - } - resp, err = q.LabelValues(ctx, req) - testutil.Ok(t, err) - testutil.Assert(t, proto.Equal(req, m1.LastLabelValuesReq), "request was not proxied properly to underlying storeAPI: %s vs %s", req, m1.LastLabelValuesReq) + expectedValues []string + expectedErr error + expectedWarningsLen int + }{ + { + title: "request all time range", + storeAPIs: storeApis, + req: &storepb.LabelValuesRequest{ + Label: "a", + PartialResponseDisabled: true, + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + }, + expectedValues: []string{"1", "2", "3", "4", "5", "6"}, + expectedWarningsLen: 1, + }, + { + title: "outside the time range of the last store client", + storeAPIs: storeApis, + req: &storepb.LabelValuesRequest{ + Label: "a", + PartialResponseDisabled: true, + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(time.Now().Add(-1 * time.Hour)), + }, + expectedValues: []string{"1", "2", "3", "4"}, + expectedWarningsLen: 1, + }, + { + title: "request all time range with limit", + storeAPIs: storeApis, + req: &storepb.LabelValuesRequest{ + Label: "a", + PartialResponseDisabled: true, + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + Limit: 2, + }, + expectedValues: []string{"1", "2"}, + expectedWarningsLen: 1, + }, + } { + if ok := t.Run(tc.title, func(t *testing.T) { + q := NewProxyStore( + nil, + nil, + func() []Client { return tc.storeAPIs }, + component.Query, + labels.EmptyLabels(), + 0*time.Second, EagerRetrieval, + ) + + ctx := context.Background() + + resp, err := q.LabelValues(ctx, tc.req) + if tc.expectedErr != nil { + testutil.NotOk(t, err) + testutil.Equals(t, tc.expectedErr.Error(), err.Error()) + return + } + testutil.Ok(t, err) + testutil.Assert(t, proto.Equal(tc.req, m1.LastLabelValuesReq), "request was not proxied properly to underlying storeAPI: %s vs %s", tc.req, m1.LastLabelValuesReq) - testutil.Equals(t, []string{"1", "2", "3", "4"}, resp.Values) - testutil.Equals(t, 1, len(resp.Warnings)) + testutil.Equals(t, tc.expectedValues, resp.Values) + testutil.Equals(t, tc.expectedWarningsLen, len(resp.Warnings), "got %v", resp.Warnings) + }); !ok { + return + } + } } func TestProxyStore_LabelNames(t *testing.T) { @@ -1651,6 +1732,33 @@ func TestProxyStore_LabelNames(t *testing.T) { expectedNames: []string{"a", "b"}, expectedWarningsLen: 0, }, + { + title: "label_names partial response disabled with limit", + storeAPIs: []Client{ + &storetestutil.TestClient{ + StoreClient: &mockedStoreAPI{ + RespLabelNames: &storepb.LabelNamesResponse{ + Names: []string{"a", "b"}, + }, + }, + }, + &storetestutil.TestClient{ + StoreClient: &mockedStoreAPI{ + RespLabelNames: &storepb.LabelNamesResponse{ + Names: []string{"a", "c", "d"}, + }, + }, + }, + }, + req: &storepb.LabelNamesRequest{ + Start: timestamp.FromTime(minTime), + End: timestamp.FromTime(maxTime), + PartialResponseDisabled: true, + Limit: 2, + }, + expectedNames: []string{"a", "b"}, + expectedWarningsLen: 0, + }, } { if ok := t.Run(tc.title, func(t *testing.T) { q := NewProxyStore( diff --git a/pkg/store/storepb/rpc.pb.go b/pkg/store/storepb/rpc.pb.go index b5e85d69d8f..3ddb507327c 100644 --- a/pkg/store/storepb/rpc.pb.go +++ b/pkg/store/storepb/rpc.pb.go @@ -305,6 +305,8 @@ type SeriesRequest struct { // NOTE(bwplotka): thanos.info.store.supports_without_replica_labels field has to return true to let client knows // server supports it. WithoutReplicaLabels []string `protobuf:"bytes,14,rep,name=without_replica_labels,json=withoutReplicaLabels,proto3" json:"without_replica_labels,omitempty"` + // limit is used to limit the number of results returned + Limit int64 `protobuf:"varint,15,opt,name=limit,proto3" json:"limit,omitempty"` } func (m *SeriesRequest) Reset() { *m = SeriesRequest{} } @@ -659,6 +661,8 @@ type LabelNamesRequest struct { Matchers []LabelMatcher `protobuf:"bytes,6,rep,name=matchers,proto3" json:"matchers"` // same as in series request. WithoutReplicaLabels []string `protobuf:"bytes,7,rep,name=without_replica_labels,json=withoutReplicaLabels,proto3" json:"without_replica_labels,omitempty"` + // limit is used to limit the number of results returned + Limit int64 `protobuf:"varint,8,opt,name=limit,proto3" json:"limit,omitempty"` } func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } @@ -750,6 +754,8 @@ type LabelValuesRequest struct { Matchers []LabelMatcher `protobuf:"bytes,7,rep,name=matchers,proto3" json:"matchers"` // same as in series request. WithoutReplicaLabels []string `protobuf:"bytes,8,rep,name=without_replica_labels,json=withoutReplicaLabels,proto3" json:"without_replica_labels,omitempty"` + // limit is used to limit the number of results returned + Limit int64 `protobuf:"varint,9,opt,name=limit,proto3" json:"limit,omitempty"` } func (m *LabelValuesRequest) Reset() { *m = LabelValuesRequest{} } @@ -850,91 +856,92 @@ func init() { func init() { proto.RegisterFile("store/storepb/rpc.proto", fileDescriptor_a938d55a388af629) } var fileDescriptor_a938d55a388af629 = []byte{ - // 1331 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x13, 0x47, - 0x14, 0xf7, 0x7a, 0xbd, 0xfe, 0xf3, 0x9c, 0xb8, 0x66, 0x30, 0x61, 0x63, 0x24, 0xc7, 0x75, 0x55, - 0xc9, 0x42, 0xd4, 0xa6, 0x06, 0x21, 0xb5, 0xe2, 0x92, 0x04, 0x43, 0xa2, 0x12, 0x53, 0xc6, 0x09, - 0x69, 0xa9, 0x2a, 0x6b, 0x6d, 0x4f, 0xd6, 0x2b, 0xec, 0xdd, 0x65, 0x67, 0xb6, 0x89, 0xaf, 0xad, - 0x7a, 0xab, 0xaa, 0xaa, 0x1f, 0xa1, 0x9f, 0x86, 0x23, 0xc7, 0xaa, 0x07, 0xd4, 0xc2, 0xbd, 0x9f, - 0xa1, 0x9a, 0x3f, 0xbb, 0xf6, 0xa6, 0x21, 0x08, 0x91, 0x4b, 0x34, 0xef, 0xf7, 0x7b, 0xf3, 0xe6, - 0xfd, 0xcf, 0x1a, 0xae, 0x52, 0xe6, 0x05, 0xa4, 0x2d, 0xfe, 0xfa, 0xc3, 0x76, 0xe0, 0x8f, 0x5a, - 0x7e, 0xe0, 0x31, 0x0f, 0x65, 0xd9, 0xc4, 0x72, 0x3d, 0x5a, 0x5d, 0x4f, 0x2a, 0xb0, 0xb9, 0x4f, - 0xa8, 0x54, 0xa9, 0x56, 0x6c, 0xcf, 0xf6, 0xc4, 0xb1, 0xcd, 0x4f, 0x0a, 0xad, 0x27, 0x2f, 0xf8, - 0x81, 0x37, 0x3b, 0x75, 0x4f, 0x99, 0x9c, 0x5a, 0x43, 0x32, 0x3d, 0x4d, 0xd9, 0x9e, 0x67, 0x4f, - 0x49, 0x5b, 0x48, 0xc3, 0xf0, 0xa8, 0x6d, 0xb9, 0x73, 0x49, 0x35, 0x3e, 0x82, 0xd5, 0xc3, 0xc0, - 0x61, 0x04, 0x13, 0xea, 0x7b, 0x2e, 0x25, 0x8d, 0x9f, 0x34, 0x58, 0x51, 0xc8, 0xf3, 0x90, 0x50, - 0x86, 0x36, 0x01, 0x98, 0x33, 0x23, 0x94, 0x04, 0x0e, 0xa1, 0xa6, 0x56, 0xd7, 0x9b, 0xc5, 0xce, - 0x35, 0x7e, 0x7b, 0x46, 0xd8, 0x84, 0x84, 0x74, 0x30, 0xf2, 0xfc, 0x79, 0x6b, 0xdf, 0x99, 0x91, - 0xbe, 0x50, 0xd9, 0xca, 0xbc, 0x78, 0xb5, 0x91, 0xc2, 0x4b, 0x97, 0xd0, 0x1a, 0x64, 0x19, 0x71, - 0x2d, 0x97, 0x99, 0xe9, 0xba, 0xd6, 0x2c, 0x60, 0x25, 0x21, 0x13, 0x72, 0x01, 0xf1, 0xa7, 0xce, - 0xc8, 0x32, 0xf5, 0xba, 0xd6, 0xd4, 0x71, 0x24, 0x36, 0x56, 0xa1, 0xb8, 0xeb, 0x1e, 0x79, 0xca, - 0x87, 0xc6, 0xef, 0x69, 0x58, 0x91, 0xb2, 0xf4, 0x12, 0x8d, 0x20, 0x2b, 0x02, 0x8d, 0x1c, 0x5a, - 0x6d, 0xc9, 0xc4, 0xb6, 0x1e, 0x72, 0x74, 0xeb, 0x2e, 0x77, 0xe1, 0xaf, 0x57, 0x1b, 0xb7, 0x6d, - 0x87, 0x4d, 0xc2, 0x61, 0x6b, 0xe4, 0xcd, 0xda, 0x52, 0xe1, 0x33, 0xc7, 0x53, 0xa7, 0xb6, 0xff, - 0xcc, 0x6e, 0x27, 0x72, 0xd6, 0x7a, 0x2a, 0x6e, 0x63, 0x65, 0x1a, 0xad, 0x43, 0x7e, 0xe6, 0xb8, - 0x03, 0x1e, 0x88, 0x70, 0x5c, 0xc7, 0xb9, 0x99, 0xe3, 0xf2, 0x48, 0x05, 0x65, 0x9d, 0x48, 0x4a, - 0xb9, 0x3e, 0xb3, 0x4e, 0x04, 0xd5, 0x86, 0x82, 0xb0, 0xba, 0x3f, 0xf7, 0x89, 0x99, 0xa9, 0x6b, - 0xcd, 0x52, 0xe7, 0x52, 0xe4, 0x5d, 0x3f, 0x22, 0xf0, 0x42, 0x07, 0xdd, 0x01, 0x10, 0x0f, 0x0e, - 0x28, 0x61, 0xd4, 0x34, 0x44, 0x3c, 0xf1, 0x0d, 0xe9, 0x52, 0x9f, 0x30, 0x95, 0xd6, 0xc2, 0x54, - 0xc9, 0xb4, 0xf1, 0x8b, 0x01, 0xab, 0x32, 0xe5, 0x51, 0xa9, 0x96, 0x1d, 0xd6, 0xde, 0xee, 0x70, - 0x3a, 0xe9, 0xf0, 0x1d, 0x4e, 0xb1, 0xd1, 0x84, 0x04, 0xd4, 0xd4, 0xc5, 0xeb, 0x95, 0x44, 0x36, - 0xf7, 0x24, 0xa9, 0x1c, 0x88, 0x75, 0x51, 0x07, 0xae, 0x70, 0x93, 0x01, 0xa1, 0xde, 0x34, 0x64, - 0x8e, 0xe7, 0x0e, 0x8e, 0x1d, 0x77, 0xec, 0x1d, 0x8b, 0xa0, 0x75, 0x7c, 0x79, 0x66, 0x9d, 0xe0, - 0x98, 0x3b, 0x14, 0x14, 0xba, 0x01, 0x60, 0xd9, 0x76, 0x40, 0x6c, 0x8b, 0x11, 0x19, 0x6b, 0xa9, - 0xb3, 0x12, 0xbd, 0xb6, 0x69, 0xdb, 0x01, 0x5e, 0xe2, 0xd1, 0x97, 0xb0, 0xee, 0x5b, 0x01, 0x73, - 0xac, 0x29, 0x7f, 0x45, 0x54, 0x7e, 0x30, 0x76, 0xa8, 0x35, 0x9c, 0x92, 0xb1, 0x99, 0xad, 0x6b, - 0xcd, 0x3c, 0xbe, 0xaa, 0x14, 0xa2, 0xce, 0xb8, 0xa7, 0x68, 0xf4, 0xdd, 0x19, 0x77, 0x29, 0x0b, - 0x2c, 0x46, 0xec, 0xb9, 0x99, 0x13, 0x65, 0xd9, 0x88, 0x1e, 0xfe, 0x3a, 0x69, 0xa3, 0xaf, 0xd4, - 0xfe, 0x67, 0x3c, 0x22, 0xd0, 0x06, 0x14, 0xe9, 0x33, 0xc7, 0x1f, 0x8c, 0x26, 0xa1, 0xfb, 0x8c, - 0x9a, 0x79, 0xe1, 0x0a, 0x70, 0x68, 0x5b, 0x20, 0xe8, 0x3a, 0x18, 0x13, 0xc7, 0x65, 0xd4, 0x2c, - 0xd4, 0x35, 0x91, 0x50, 0x39, 0x81, 0xad, 0x68, 0x02, 0x5b, 0x9b, 0xee, 0x1c, 0x4b, 0x15, 0x84, - 0x20, 0x43, 0x19, 0xf1, 0x4d, 0x10, 0x69, 0x13, 0x67, 0x54, 0x01, 0x23, 0xb0, 0x5c, 0x9b, 0x98, - 0x45, 0x01, 0x4a, 0x01, 0xdd, 0x82, 0xe2, 0xf3, 0x90, 0x04, 0xf3, 0x81, 0xb4, 0xbd, 0x22, 0x6c, - 0xa3, 0x28, 0x8a, 0xc7, 0x9c, 0xda, 0xe1, 0x0c, 0x86, 0xe7, 0xf1, 0x19, 0xdd, 0x04, 0xa0, 0x13, - 0x2b, 0x18, 0x0f, 0x1c, 0xf7, 0xc8, 0x33, 0x57, 0xc5, 0x9d, 0x45, 0x43, 0x72, 0x46, 0x4c, 0x56, - 0x81, 0x46, 0x47, 0x74, 0x1b, 0xd6, 0x8e, 0x1d, 0x36, 0xf1, 0x42, 0x36, 0x50, 0xf3, 0x38, 0x50, - 0xc3, 0x56, 0xaa, 0xeb, 0xcd, 0x02, 0xae, 0x28, 0x16, 0x4b, 0x52, 0x34, 0x09, 0x6d, 0xfc, 0xa1, - 0x01, 0x2c, 0x5c, 0x10, 0x29, 0x62, 0xc4, 0x1f, 0xcc, 0x9c, 0xe9, 0xd4, 0xa1, 0xaa, 0x1d, 0x81, - 0x43, 0x7b, 0x02, 0x41, 0x75, 0xc8, 0x1c, 0x85, 0xee, 0x48, 0x74, 0x63, 0x71, 0xd1, 0x04, 0xf7, - 0x43, 0x77, 0x84, 0x05, 0x83, 0x6e, 0x40, 0xde, 0x0e, 0xbc, 0xd0, 0x77, 0x5c, 0x5b, 0xf4, 0x54, - 0xb1, 0x53, 0x8e, 0xb4, 0x1e, 0x28, 0x1c, 0xc7, 0x1a, 0xe8, 0x93, 0x28, 0x65, 0x86, 0x50, 0x8d, - 0x37, 0x02, 0xe6, 0xa0, 0xca, 0x60, 0xe3, 0x18, 0x0a, 0x71, 0xc8, 0xc2, 0x45, 0x95, 0x99, 0x31, - 0x39, 0x89, 0x5d, 0x94, 0xfc, 0x98, 0x9c, 0xa0, 0x8f, 0x61, 0x85, 0x79, 0xcc, 0x9a, 0x0e, 0x04, - 0x46, 0xd5, 0xe0, 0x14, 0x05, 0x26, 0xcc, 0x50, 0x54, 0x82, 0xf4, 0x70, 0x2e, 0x56, 0x40, 0x1e, - 0xa7, 0x87, 0x73, 0xbe, 0xea, 0x54, 0xae, 0x32, 0x22, 0x57, 0x4a, 0x6a, 0x54, 0x21, 0xc3, 0x23, - 0xe3, 0xc5, 0x76, 0x2d, 0x35, 0x9e, 0x05, 0x2c, 0xce, 0x8d, 0x0e, 0xe4, 0xa3, 0x78, 0x94, 0x3d, - 0xed, 0x0c, 0x7b, 0x7a, 0xc2, 0xde, 0x06, 0x18, 0x22, 0x30, 0xae, 0x90, 0x48, 0xb1, 0x92, 0x1a, - 0xbf, 0x6a, 0x50, 0x8a, 0xb6, 0x83, 0x5a, 0x9a, 0x4d, 0xc8, 0xc6, 0x5b, 0x9c, 0xa7, 0xa8, 0x14, - 0x77, 0x81, 0x40, 0x77, 0x52, 0x58, 0xf1, 0xa8, 0x0a, 0xb9, 0x63, 0x2b, 0x70, 0x79, 0xe2, 0xc5, - 0xc6, 0xde, 0x49, 0xe1, 0x08, 0x40, 0x37, 0xa2, 0xd6, 0xd6, 0xdf, 0xde, 0xda, 0x3b, 0x29, 0xd5, - 0xdc, 0x5b, 0x79, 0xc8, 0x06, 0x84, 0x86, 0x53, 0xd6, 0xf8, 0x37, 0x0d, 0x97, 0x44, 0xab, 0xf4, - 0xac, 0xd9, 0x62, 0x65, 0x9d, 0x3b, 0xe2, 0xda, 0x07, 0x8c, 0x78, 0xfa, 0x03, 0x47, 0xbc, 0x02, - 0x06, 0x65, 0x56, 0xc0, 0xd4, 0x7a, 0x97, 0x02, 0x2a, 0x83, 0x4e, 0xdc, 0xb1, 0xda, 0x70, 0xfc, - 0xb8, 0x98, 0x74, 0xe3, 0xdd, 0x93, 0xbe, 0xbc, 0x69, 0xb3, 0xef, 0xb1, 0x69, 0xdf, 0x3e, 0x90, - 0xb9, 0x73, 0x06, 0x32, 0x00, 0xb4, 0x9c, 0x6f, 0xd5, 0x04, 0x15, 0x30, 0x78, 0xd3, 0xc9, 0x7f, - 0x9c, 0x05, 0x2c, 0x05, 0x54, 0x85, 0xbc, 0xaa, 0x2f, 0xef, 0x72, 0x4e, 0xc4, 0xf2, 0x22, 0x42, - 0xfd, 0x9d, 0x11, 0x36, 0x7e, 0xd6, 0xd5, 0xa3, 0x4f, 0xac, 0x69, 0xb8, 0xa8, 0x72, 0x05, 0x0c, - 0xe1, 0xb0, 0x6a, 0x7b, 0x29, 0x9c, 0x5f, 0xfb, 0xf4, 0x07, 0xd4, 0x5e, 0xbf, 0xa8, 0xda, 0x67, - 0xce, 0xa8, 0xbd, 0x71, 0x46, 0xed, 0xb3, 0xef, 0x57, 0xfb, 0xdc, 0x85, 0xd4, 0x3e, 0x7f, 0x4e, - 0xed, 0x43, 0xb8, 0x9c, 0x28, 0x83, 0x2a, 0xfe, 0x1a, 0x64, 0x7f, 0x10, 0x88, 0xaa, 0xbe, 0x92, - 0x2e, 0xaa, 0xfc, 0xd7, 0xbf, 0x87, 0x42, 0xfc, 0x89, 0x83, 0x8a, 0x90, 0x3b, 0xe8, 0x7d, 0xd5, - 0x7b, 0x74, 0xd8, 0x2b, 0xa7, 0x50, 0x01, 0x8c, 0xc7, 0x07, 0x5d, 0xfc, 0x6d, 0x59, 0x43, 0x79, - 0xc8, 0xe0, 0x83, 0x87, 0xdd, 0x72, 0x9a, 0x6b, 0xf4, 0x77, 0xef, 0x75, 0xb7, 0x37, 0x71, 0x59, - 0xe7, 0x1a, 0xfd, 0xfd, 0x47, 0xb8, 0x5b, 0xce, 0x70, 0x1c, 0x77, 0xb7, 0xbb, 0xbb, 0x4f, 0xba, - 0x65, 0x83, 0xe3, 0xf7, 0xba, 0x5b, 0x07, 0x0f, 0xca, 0xd9, 0xeb, 0x5b, 0x90, 0xe1, 0xdf, 0x08, - 0x28, 0x07, 0x3a, 0xde, 0x3c, 0x94, 0x56, 0xb7, 0x1f, 0x1d, 0xf4, 0xf6, 0xcb, 0x1a, 0xc7, 0xfa, - 0x07, 0x7b, 0xe5, 0x34, 0x3f, 0xec, 0xed, 0xf6, 0xca, 0xba, 0x38, 0x6c, 0x7e, 0x23, 0xcd, 0x09, - 0xad, 0x2e, 0x2e, 0x1b, 0x9d, 0x1f, 0xd3, 0x60, 0x08, 0x1f, 0xd1, 0xe7, 0x90, 0x11, 0xff, 0x06, - 0x2e, 0x47, 0x75, 0x58, 0xfa, 0xe2, 0xac, 0x56, 0x92, 0xa0, 0xca, 0xdf, 0x17, 0x90, 0x95, 0xbb, - 0x12, 0x5d, 0x49, 0xee, 0xce, 0xe8, 0xda, 0xda, 0x69, 0x58, 0x5e, 0xbc, 0xa9, 0xa1, 0x6d, 0x80, - 0xc5, 0x34, 0xa2, 0xf5, 0x44, 0xed, 0x97, 0x37, 0x62, 0xb5, 0x7a, 0x16, 0xa5, 0xde, 0xbf, 0x0f, - 0xc5, 0xa5, 0xb2, 0xa2, 0xa4, 0x6a, 0x62, 0xe4, 0xaa, 0xd7, 0xce, 0xe4, 0xa4, 0x9d, 0x4e, 0x0f, - 0x4a, 0xe2, 0x1b, 0x9f, 0xcf, 0x92, 0x4c, 0xc6, 0x5d, 0x28, 0x62, 0x32, 0xf3, 0x18, 0x11, 0x38, - 0x8a, 0xc3, 0x5f, 0xfe, 0x29, 0x50, 0xbd, 0x72, 0x0a, 0x55, 0x3f, 0x19, 0x52, 0x5b, 0x9f, 0xbe, - 0xf8, 0xa7, 0x96, 0x7a, 0xf1, 0xba, 0xa6, 0xbd, 0x7c, 0x5d, 0xd3, 0xfe, 0x7e, 0x5d, 0xd3, 0x7e, - 0x7b, 0x53, 0x4b, 0xbd, 0x7c, 0x53, 0x4b, 0xfd, 0xf9, 0xa6, 0x96, 0x7a, 0x9a, 0x53, 0xbf, 0x5a, - 0x86, 0x59, 0xd1, 0x33, 0xb7, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xa0, 0x14, 0xa2, 0x0f, 0x1f, - 0x0d, 0x00, 0x00, + // 1351 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0xdb, 0x46, + 0x13, 0x16, 0x45, 0x51, 0x1f, 0x23, 0x5b, 0x51, 0x36, 0x8a, 0x43, 0x2b, 0x80, 0xac, 0x57, 0x2f, + 0x0a, 0x08, 0x41, 0x2a, 0xa5, 0x4a, 0x10, 0xa0, 0x45, 0x2e, 0xb6, 0xa3, 0xc4, 0x46, 0x63, 0xa5, + 0x59, 0xd9, 0x71, 0x9b, 0xa2, 0x10, 0x28, 0x69, 0x4d, 0x11, 0xa1, 0x48, 0x86, 0xbb, 0xac, 0xad, + 0x6b, 0xdb, 0x73, 0x51, 0xf4, 0xda, 0x5b, 0x7e, 0x4d, 0x8e, 0x39, 0x16, 0x3d, 0x04, 0x6d, 0xf2, + 0x47, 0x8a, 0xfd, 0xa0, 0x24, 0xba, 0xce, 0x17, 0xe2, 0x8b, 0xb1, 0xf3, 0x3c, 0xb3, 0xb3, 0xb3, + 0xb3, 0xcf, 0x8c, 0x45, 0xb8, 0x42, 0x99, 0x1f, 0x92, 0xb6, 0xf8, 0x1b, 0x0c, 0xdb, 0x61, 0x30, + 0x6a, 0x05, 0xa1, 0xcf, 0x7c, 0x94, 0x65, 0x13, 0xcb, 0xf3, 0x69, 0x75, 0x3d, 0xe9, 0xc0, 0x66, + 0x01, 0xa1, 0xd2, 0xa5, 0x5a, 0xb1, 0x7d, 0xdb, 0x17, 0xcb, 0x36, 0x5f, 0x29, 0xb4, 0x9e, 0xdc, + 0x10, 0x84, 0xfe, 0xf4, 0xd4, 0x3e, 0x15, 0xd2, 0xb5, 0x86, 0xc4, 0x3d, 0x4d, 0xd9, 0xbe, 0x6f, + 0xbb, 0xa4, 0x2d, 0xac, 0x61, 0x74, 0xd4, 0xb6, 0xbc, 0x99, 0xa4, 0x1a, 0x17, 0x60, 0xf5, 0x30, + 0x74, 0x18, 0xc1, 0x84, 0x06, 0xbe, 0x47, 0x49, 0xe3, 0x67, 0x0d, 0x56, 0x14, 0xf2, 0x2c, 0x22, + 0x94, 0xa1, 0x4d, 0x00, 0xe6, 0x4c, 0x09, 0x25, 0xa1, 0x43, 0xa8, 0xa9, 0xd5, 0xf5, 0x66, 0xb1, + 0x73, 0x95, 0xef, 0x9e, 0x12, 0x36, 0x21, 0x11, 0x1d, 0x8c, 0xfc, 0x60, 0xd6, 0xda, 0x77, 0xa6, + 0xa4, 0x2f, 0x5c, 0xb6, 0x32, 0x2f, 0x5e, 0x6d, 0xa4, 0xf0, 0xd2, 0x26, 0xb4, 0x06, 0x59, 0x46, + 0x3c, 0xcb, 0x63, 0x66, 0xba, 0xae, 0x35, 0x0b, 0x58, 0x59, 0xc8, 0x84, 0x5c, 0x48, 0x02, 0xd7, + 0x19, 0x59, 0xa6, 0x5e, 0xd7, 0x9a, 0x3a, 0x8e, 0xcd, 0xc6, 0x2a, 0x14, 0x77, 0xbd, 0x23, 0x5f, + 0xe5, 0xd0, 0xf8, 0x3d, 0x0d, 0x2b, 0xd2, 0x96, 0x59, 0xa2, 0x11, 0x64, 0xc5, 0x45, 0xe3, 0x84, + 0x56, 0x5b, 0xb2, 0xb0, 0xad, 0x07, 0x1c, 0xdd, 0xba, 0xc3, 0x53, 0xf8, 0xeb, 0xd5, 0xc6, 0x2d, + 0xdb, 0x61, 0x93, 0x68, 0xd8, 0x1a, 0xf9, 0xd3, 0xb6, 0x74, 0xf8, 0xdc, 0xf1, 0xd5, 0xaa, 0x1d, + 0x3c, 0xb5, 0xdb, 0x89, 0x9a, 0xb5, 0x9e, 0x88, 0xdd, 0x58, 0x85, 0x46, 0xeb, 0x90, 0x9f, 0x3a, + 0xde, 0x80, 0x5f, 0x44, 0x24, 0xae, 0xe3, 0xdc, 0xd4, 0xf1, 0xf8, 0x4d, 0x05, 0x65, 0x9d, 0x48, + 0x4a, 0xa5, 0x3e, 0xb5, 0x4e, 0x04, 0xd5, 0x86, 0x82, 0x88, 0xba, 0x3f, 0x0b, 0x88, 0x99, 0xa9, + 0x6b, 0xcd, 0x52, 0xe7, 0x62, 0x9c, 0x5d, 0x3f, 0x26, 0xf0, 0xc2, 0x07, 0xdd, 0x06, 0x10, 0x07, + 0x0e, 0x28, 0x61, 0xd4, 0x34, 0xc4, 0x7d, 0xe6, 0x3b, 0x64, 0x4a, 0x7d, 0xc2, 0x54, 0x59, 0x0b, + 0xae, 0xb2, 0x69, 0xe3, 0xb9, 0x01, 0xab, 0xb2, 0xe4, 0xf1, 0x53, 0x2d, 0x27, 0xac, 0xbd, 0x3d, + 0xe1, 0x74, 0x32, 0xe1, 0xdb, 0x9c, 0x62, 0xa3, 0x09, 0x09, 0xa9, 0xa9, 0x8b, 0xd3, 0x2b, 0x89, + 0x6a, 0xee, 0x49, 0x52, 0x25, 0x30, 0xf7, 0x45, 0x1d, 0xb8, 0xcc, 0x43, 0x86, 0x84, 0xfa, 0x6e, + 0xc4, 0x1c, 0xdf, 0x1b, 0x1c, 0x3b, 0xde, 0xd8, 0x3f, 0x16, 0x97, 0xd6, 0xf1, 0xa5, 0xa9, 0x75, + 0x82, 0xe7, 0xdc, 0xa1, 0xa0, 0xd0, 0x75, 0x00, 0xcb, 0xb6, 0x43, 0x62, 0x5b, 0x8c, 0xc8, 0xbb, + 0x96, 0x3a, 0x2b, 0xf1, 0x69, 0x9b, 0xb6, 0x1d, 0xe2, 0x25, 0x1e, 0x7d, 0x05, 0xeb, 0x81, 0x15, + 0x32, 0xc7, 0x72, 0xf9, 0x29, 0xe2, 0xe5, 0x07, 0x63, 0x87, 0x5a, 0x43, 0x97, 0x8c, 0xcd, 0x6c, + 0x5d, 0x6b, 0xe6, 0xf1, 0x15, 0xe5, 0x10, 0x2b, 0xe3, 0xae, 0xa2, 0xd1, 0xf7, 0x67, 0xec, 0xa5, + 0x2c, 0xb4, 0x18, 0xb1, 0x67, 0x66, 0x4e, 0x3c, 0xcb, 0x46, 0x7c, 0xf0, 0x37, 0xc9, 0x18, 0x7d, + 0xe5, 0xf6, 0x9f, 0xe0, 0x31, 0x81, 0x36, 0xa0, 0x48, 0x9f, 0x3a, 0xc1, 0x60, 0x34, 0x89, 0xbc, + 0xa7, 0xd4, 0xcc, 0x8b, 0x54, 0x80, 0x43, 0xdb, 0x02, 0x41, 0xd7, 0xc0, 0x98, 0x38, 0x1e, 0xa3, + 0x66, 0xa1, 0xae, 0x89, 0x82, 0xca, 0x0e, 0x6c, 0xc5, 0x1d, 0xd8, 0xda, 0xf4, 0x66, 0x58, 0xba, + 0x20, 0x04, 0x19, 0xca, 0x48, 0x60, 0x82, 0x28, 0x9b, 0x58, 0xa3, 0x0a, 0x18, 0xa1, 0xe5, 0xd9, + 0xc4, 0x2c, 0x0a, 0x50, 0x1a, 0xe8, 0x26, 0x14, 0x9f, 0x45, 0x24, 0x9c, 0x0d, 0x64, 0xec, 0x15, + 0x11, 0x1b, 0xc5, 0xb7, 0x78, 0xc4, 0xa9, 0x1d, 0xce, 0x60, 0x78, 0x36, 0x5f, 0xa3, 0x1b, 0x00, + 0x74, 0x62, 0x85, 0xe3, 0x81, 0xe3, 0x1d, 0xf9, 0xe6, 0xaa, 0xd8, 0xb3, 0x10, 0x24, 0x67, 0x44, + 0x67, 0x15, 0x68, 0xbc, 0x44, 0xb7, 0x60, 0xed, 0xd8, 0x61, 0x13, 0x3f, 0x62, 0x03, 0xd5, 0x8f, + 0x03, 0xd5, 0x6c, 0xa5, 0xba, 0xde, 0x2c, 0xe0, 0x8a, 0x62, 0xb1, 0x24, 0x1f, 0xc8, 0x6e, 0xa9, + 0x80, 0xe1, 0x3a, 0x53, 0x87, 0x99, 0x17, 0x64, 0xca, 0xc2, 0x68, 0x3c, 0xd7, 0x00, 0x16, 0x89, + 0x89, 0xc2, 0x31, 0x12, 0x0c, 0xa6, 0x8e, 0xeb, 0x3a, 0x54, 0x89, 0x14, 0x38, 0xb4, 0x27, 0x10, + 0x54, 0x87, 0xcc, 0x51, 0xe4, 0x8d, 0x84, 0x46, 0x8b, 0x0b, 0x69, 0xdc, 0x8b, 0xbc, 0x11, 0x16, + 0x0c, 0xba, 0x0e, 0x79, 0x3b, 0xf4, 0xa3, 0xc0, 0xf1, 0x6c, 0xa1, 0xb4, 0x62, 0xa7, 0x1c, 0x7b, + 0xdd, 0x57, 0x38, 0x9e, 0x7b, 0xa0, 0xff, 0xc7, 0x85, 0x34, 0x84, 0xeb, 0x7c, 0x4e, 0x60, 0x0e, + 0xaa, 0xba, 0x36, 0x8e, 0xa1, 0x30, 0x2f, 0x84, 0x48, 0x51, 0xd5, 0x6b, 0x4c, 0x4e, 0xe6, 0x29, + 0x4a, 0x7e, 0x4c, 0x4e, 0xd0, 0xff, 0x60, 0x85, 0xf9, 0xcc, 0x72, 0x07, 0x02, 0xa3, 0xaa, 0x9d, + 0x8a, 0x02, 0x13, 0x61, 0x28, 0x2a, 0x41, 0x7a, 0x38, 0x13, 0x83, 0x21, 0x8f, 0xd3, 0xc3, 0x19, + 0x1f, 0x80, 0xaa, 0x82, 0x19, 0x51, 0x41, 0x65, 0x35, 0xaa, 0x90, 0xe1, 0x37, 0xe3, 0x12, 0xf0, + 0x2c, 0xd5, 0xb4, 0x05, 0x2c, 0xd6, 0x8d, 0x0e, 0xe4, 0xe3, 0xfb, 0xa8, 0x78, 0xda, 0x19, 0xf1, + 0xf4, 0x44, 0xbc, 0x0d, 0x30, 0xc4, 0xc5, 0xb8, 0x43, 0xa2, 0xc4, 0xca, 0x6a, 0xfc, 0xaa, 0x41, + 0x29, 0x9e, 0x19, 0x6a, 0x94, 0x36, 0x21, 0x3b, 0x9f, 0xed, 0xbc, 0x44, 0xa5, 0xb9, 0x36, 0x04, + 0xba, 0x93, 0xc2, 0x8a, 0x47, 0x55, 0xc8, 0x1d, 0x5b, 0xa1, 0xc7, 0x0b, 0x2f, 0xe6, 0xf8, 0x4e, + 0x0a, 0xc7, 0x00, 0xba, 0x1e, 0x0b, 0x5e, 0x7f, 0xbb, 0xe0, 0x77, 0x52, 0x4a, 0xf2, 0x5b, 0x79, + 0xc8, 0x86, 0x84, 0x46, 0x2e, 0x6b, 0xfc, 0xa2, 0xc3, 0x45, 0x21, 0xa0, 0x9e, 0x35, 0x5d, 0x0c, + 0xb2, 0x77, 0x36, 0xbe, 0xf6, 0x09, 0x8d, 0x9f, 0xfe, 0xc4, 0xc6, 0xaf, 0x80, 0x41, 0x99, 0x15, + 0x32, 0x35, 0xf4, 0xa5, 0x81, 0xca, 0xa0, 0x13, 0x6f, 0xac, 0xe6, 0x1e, 0x5f, 0x2e, 0xfa, 0xdf, + 0x78, 0x7f, 0xff, 0x2f, 0xcf, 0xdf, 0xec, 0x47, 0xcc, 0xdf, 0xb7, 0xb7, 0x69, 0xee, 0x43, 0xda, + 0x34, 0xbf, 0xdc, 0xa6, 0x21, 0xa0, 0xe5, 0x57, 0x50, 0xd2, 0xa8, 0x80, 0xc1, 0xa5, 0x28, 0xff, + 0xc9, 0x16, 0xb0, 0x34, 0x50, 0x15, 0xf2, 0xea, 0xd5, 0xb9, 0xf6, 0x39, 0x31, 0xb7, 0x17, 0xf7, + 0xd6, 0xdf, 0x7b, 0xef, 0xc6, 0x1f, 0xba, 0x3a, 0xf4, 0xb1, 0xe5, 0x46, 0x8b, 0xb7, 0xe7, 0x09, + 0x72, 0x54, 0x35, 0x83, 0x34, 0xde, 0xad, 0x88, 0xf4, 0x27, 0x28, 0x42, 0x3f, 0x2f, 0x45, 0x64, + 0xce, 0x50, 0x84, 0x71, 0x86, 0x22, 0xb2, 0x1f, 0xa7, 0x88, 0xdc, 0xb9, 0x28, 0x22, 0xff, 0x21, + 0x8a, 0x28, 0x2c, 0x2b, 0x22, 0x82, 0x4b, 0x89, 0xc7, 0x51, 0x92, 0x58, 0x83, 0xec, 0x8f, 0x02, + 0x51, 0x9a, 0x50, 0xd6, 0x79, 0x89, 0xe2, 0xda, 0x0f, 0x50, 0x98, 0xff, 0x48, 0x42, 0x45, 0xc8, + 0x1d, 0xf4, 0xbe, 0xee, 0x3d, 0x3c, 0xec, 0x95, 0x53, 0xa8, 0x00, 0xc6, 0xa3, 0x83, 0x2e, 0xfe, + 0xae, 0xac, 0xa1, 0x3c, 0x64, 0xf0, 0xc1, 0x83, 0x6e, 0x39, 0xcd, 0x3d, 0xfa, 0xbb, 0x77, 0xbb, + 0xdb, 0x9b, 0xb8, 0xac, 0x73, 0x8f, 0xfe, 0xfe, 0x43, 0xdc, 0x2d, 0x67, 0x38, 0x8e, 0xbb, 0xdb, + 0xdd, 0xdd, 0xc7, 0xdd, 0xb2, 0xc1, 0xf1, 0xbb, 0xdd, 0xad, 0x83, 0xfb, 0xe5, 0xec, 0xb5, 0x2d, + 0xc8, 0xf0, 0x5f, 0x19, 0x28, 0x07, 0x3a, 0xde, 0x3c, 0x94, 0x51, 0xb7, 0x1f, 0x1e, 0xf4, 0xf6, + 0xcb, 0x1a, 0xc7, 0xfa, 0x07, 0x7b, 0xe5, 0x34, 0x5f, 0xec, 0xed, 0xf6, 0xca, 0xba, 0x58, 0x6c, + 0x7e, 0x2b, 0xc3, 0x09, 0xaf, 0x2e, 0x2e, 0x1b, 0x9d, 0x9f, 0xd2, 0x60, 0x88, 0x1c, 0xd1, 0x17, + 0x90, 0x11, 0xff, 0x32, 0x2e, 0xc5, 0xaf, 0xb3, 0xf4, 0x9b, 0xb5, 0x5a, 0x49, 0x82, 0xaa, 0x7e, + 0x5f, 0x42, 0x56, 0xce, 0x55, 0x74, 0x39, 0x39, 0x67, 0xe3, 0x6d, 0x6b, 0xa7, 0x61, 0xb9, 0xf1, + 0x86, 0x86, 0xb6, 0x01, 0x16, 0x3d, 0x8a, 0xd6, 0x13, 0x8a, 0x58, 0x9e, 0x9e, 0xd5, 0xea, 0x59, + 0x94, 0x3a, 0xff, 0x1e, 0x14, 0x97, 0x9e, 0x15, 0x25, 0x5d, 0x13, 0x8d, 0x58, 0xbd, 0x7a, 0x26, + 0x27, 0xe3, 0x74, 0x7a, 0x50, 0x12, 0x5f, 0x09, 0xbc, 0xc3, 0x64, 0x31, 0xee, 0x40, 0x11, 0x93, + 0xa9, 0xcf, 0x88, 0xc0, 0xd1, 0xfc, 0xfa, 0xcb, 0x1f, 0x13, 0xd5, 0xcb, 0xa7, 0x50, 0xf5, 0xd1, + 0x91, 0xda, 0xfa, 0xec, 0xc5, 0x3f, 0xb5, 0xd4, 0x8b, 0xd7, 0x35, 0xed, 0xe5, 0xeb, 0x9a, 0xf6, + 0xf7, 0xeb, 0x9a, 0xf6, 0xdb, 0x9b, 0x5a, 0xea, 0xe5, 0x9b, 0x5a, 0xea, 0xcf, 0x37, 0xb5, 0xd4, + 0x93, 0x9c, 0xfa, 0xee, 0x19, 0x66, 0x85, 0x66, 0x6e, 0xfe, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x22, + 0xcb, 0x99, 0x7e, 0x61, 0x0d, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1436,6 +1443,11 @@ func (m *SeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Limit != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x78 + } if len(m.WithoutReplicaLabels) > 0 { for iNdEx := len(m.WithoutReplicaLabels) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.WithoutReplicaLabels[iNdEx]) @@ -1890,6 +1902,11 @@ func (m *LabelNamesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Limit != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x40 + } if len(m.WithoutReplicaLabels) > 0 { for iNdEx := len(m.WithoutReplicaLabels) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.WithoutReplicaLabels[iNdEx]) @@ -2026,6 +2043,11 @@ func (m *LabelValuesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Limit != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x48 + } if len(m.WithoutReplicaLabels) > 0 { for iNdEx := len(m.WithoutReplicaLabels) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.WithoutReplicaLabels[iNdEx]) @@ -2291,6 +2313,9 @@ func (m *SeriesRequest) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } + if m.Limit != 0 { + n += 1 + sovRpc(uint64(m.Limit)) + } return n } @@ -2465,6 +2490,9 @@ func (m *LabelNamesRequest) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } + if m.Limit != 0 { + n += 1 + sovRpc(uint64(m.Limit)) + } return n } @@ -2531,6 +2559,9 @@ func (m *LabelValuesRequest) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } + if m.Limit != 0 { + n += 1 + sovRpc(uint64(m.Limit)) + } return n } @@ -3401,6 +3432,25 @@ func (m *SeriesRequest) Unmarshal(dAtA []byte) error { } m.WithoutReplicaLabels = append(m.WithoutReplicaLabels, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 15: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -4352,6 +4402,25 @@ func (m *LabelNamesRequest) Unmarshal(dAtA []byte) error { } m.WithoutReplicaLabels = append(m.WithoutReplicaLabels, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -4763,6 +4832,25 @@ func (m *LabelValuesRequest) Unmarshal(dAtA []byte) error { } m.WithoutReplicaLabels = append(m.WithoutReplicaLabels, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index 2a9e9e3eaf6..a15e5b6f8e3 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -137,6 +137,9 @@ message SeriesRequest { // NOTE(bwplotka): thanos.info.store.supports_without_replica_labels field has to return true to let client knows // server supports it. repeated string without_replica_labels = 14; + + // limit is used to limit the number of results returned + int64 limit = 15; } // QueryHints represents hints from PromQL that might help to @@ -235,6 +238,9 @@ message LabelNamesRequest { // same as in series request. repeated string without_replica_labels = 7; + + // limit is used to limit the number of results returned + int64 limit = 8; } message LabelNamesResponse { @@ -268,6 +274,9 @@ message LabelValuesRequest { // same as in series request. repeated string without_replica_labels = 8; + + // limit is used to limit the number of results returned + int64 limit = 9; } message LabelValuesResponse { diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index b19c131b545..6dd18af0a8b 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -220,7 +220,12 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, seriesSrv storepb.Store_Ser defer runutil.CloseWithLogOnErr(s.logger, q, "close tsdb chunk querier series") } - set := q.Select(srv.Context(), true, nil, matchers...) + hints := &storage.SelectHints{ + Start: r.MinTime, + End: r.MaxTime, + Limit: int(r.Limit), + } + set := q.Select(srv.Context(), true, hints, matchers...) shardMatcher := r.ShardInfo.Matcher(&s.buffers) defer shardMatcher.Close() @@ -328,7 +333,10 @@ func (s *TSDBStore) LabelNames(ctx context.Context, r *storepb.LabelNamesRequest } defer runutil.CloseWithLogOnErr(s.logger, q, "close tsdb querier label names") - res, _, err := q.LabelNames(ctx, nil, matchers...) + hints := &storage.LabelHints{ + Limit: int(r.Limit), + } + res, _, err := q.LabelNames(ctx, hints, matchers...) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -396,6 +404,7 @@ func (s *TSDBStore) LabelValues(ctx context.Context, r *storepb.LabelValuesReque Start: r.Start, End: r.End, Func: "series", + Limit: int(r.Limit), } set := q.Select(ctx, false, hints, matchers...) @@ -405,7 +414,10 @@ func (s *TSDBStore) LabelValues(ctx context.Context, r *storepb.LabelValuesReque return &storepb.LabelValuesResponse{}, nil } - res, _, err := q.LabelValues(ctx, r.Label, nil, matchers...) + hints := &storage.LabelHints{ + Limit: int(r.Limit), + } + res, _, err := q.LabelValues(ctx, r.Label, hints, matchers...) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/test/e2e/exemplars_api_test.go b/test/e2e/exemplars_api_test.go index 8efba21e019..60c423f6670 100644 --- a/test/e2e/exemplars_api_test.go +++ b/test/e2e/exemplars_api_test.go @@ -89,7 +89,7 @@ config: end := timestamp.FromTime(now.Add(time.Hour)) // Send HTTP requests to thanos query to trigger exemplars. - labelNames(t, ctx, q.Endpoint("http"), nil, start, end, func(res []string) bool { return true }) + labelNames(t, ctx, q.Endpoint("http"), nil, start, end, 0, func(res []string) bool { return true }) t.Run("Basic exemplars query", func(t *testing.T) { queryExemplars(t, ctx, q.Endpoint("http"), `http_request_duration_seconds_bucket{handler="label_names"}`, start, end, exemplarsOnExpectedSeries(map[string]string{ diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 108013cdbfa..772241d34b5 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -270,7 +270,7 @@ func TestQueryFrontend(t *testing.T) { t.Run("query frontend splitting works for labels names API", func(t *testing.T) { // LabelNames and LabelValues API should still work via query frontend. - labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), 0, func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -289,7 +289,7 @@ func TestQueryFrontend(t *testing.T) { e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), 0, func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -310,7 +310,7 @@ func TestQueryFrontend(t *testing.T) { }) t.Run("query frontend splitting works for labels values API", func(t *testing.T) { - labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -329,7 +329,7 @@ func TestQueryFrontend(t *testing.T) { e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -357,6 +357,7 @@ func TestQueryFrontend(t *testing.T) { []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), + 0, func(res []map[string]string) bool { if len(res) != 1 { return false @@ -395,6 +396,7 @@ func TestQueryFrontend(t *testing.T) { []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), + 0, func(res []map[string]string) bool { if len(res) != 1 { return false diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 8ef76001365..96e717744d1 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -443,17 +443,17 @@ func TestQueryLabelNames(t *testing.T) { t.Cleanup(cancel) now := time.Now() - labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) > 0 }) // Outside time range. - labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), 0, func(res []string) bool { return len(res) == 0 }) labelNames(t, ctx, q.Endpoint("http"), []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "up"}}, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { // Expected result: [__name__, instance, job, prometheus, replica, receive, tenant_id] // Pre-labelnames pushdown we've done Select() over all series and picked out the label names hence they all had external labels. // With labelnames pushdown we had to extend the LabelNames() call to enrich the response with the external labelset when there is more than one label. @@ -461,9 +461,17 @@ func TestQueryLabelNames(t *testing.T) { }, ) + // With limit + labelNames(t, ctx, q.Endpoint("http"), []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "up"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3, func(res []string) bool { + // Same test as above but requesting limited results. + return len(res) == 3 + }, + ) + // There is no matched series. labelNames(t, ctx, q.Endpoint("http"), []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "foobar"}}, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 0 }, ) @@ -490,28 +498,71 @@ func TestQueryLabelValues(t *testing.T) { t.Cleanup(cancel) now := time.Now() - labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) // Outside time range. - labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), 0, func(res []string) bool { return len(res) == 0 }) labelValues(t, ctx, q.Endpoint("http"), "__name__", []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "up"}}, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 1 && res[0] == "up" }, ) + labelValues(t, ctx, q.Endpoint("http"), "__name__", []*labels.Matcher{}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 2, func(res []string) bool { + return len(res) == 2 + }, + ) + labelValues(t, ctx, q.Endpoint("http"), "__name__", []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "foobar"}}, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 0 }, ) } +func TestQuerySeries(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("series") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + receiver := e2ethanos.NewReceiveBuilder(e, "1").WithIngestionEnabled().Init() + testutil.Ok(t, e2e.StartAndWaitReady(receiver)) + + prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "alone", e2ethanos.DefaultPromConfig("prom-alone", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "") + prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", e2ethanos.DefaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "") + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + + q := e2ethanos.NewQuerierBuilder(e, "1", sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), receiver.InternalEndpoint("grpc")).Init() + testutil.Ok(t, e2e.StartAndWaitReady(q)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + + now := time.Now() + + // Without limit + series(t, ctx, q.Endpoint("http"), []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "up"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []map[string]string) bool { + return len(res) == 3 + }, + ) + + // With limit + series(t, ctx, q.Endpoint("http"), []*labels.Matcher{{Type: labels.MatchEqual, Name: "__name__", Value: "up"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 1, func(res []map[string]string) bool { + return len(res) == 1 + }, + ) +} + func TestQueryWithAuthorizedSidecar(t *testing.T) { t.Parallel() @@ -649,7 +700,7 @@ config: end := timestamp.FromTime(now.Add(time.Hour)) // Send HTTP requests to thanos query to trigger exemplars. - labelNames(t, ctx, q.Endpoint("http"), nil, start, end, func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, start, end, 0, func(res []string) bool { return true }) @@ -1563,13 +1614,13 @@ func queryAndAssert(t *testing.T, ctx context.Context, addr string, q func() str testutil.Equals(t, expected, result) } -func labelNames(t *testing.T, ctx context.Context, addr string, matchers []*labels.Matcher, start, end int64, check func(res []string) bool) { +func labelNames(t *testing.T, ctx context.Context, addr string, matchers []*labels.Matcher, start, end int64, limit int, check func(res []string) bool) { t.Helper() logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) testutil.Ok(t, runutil.RetryWithLog(logger, 2*time.Second, ctx.Done(), func() error { - res, err := promclient.NewDefaultClient().LabelNamesInGRPC(ctx, urlParse(t, "http://"+addr), matchers, start, end) + res, err := promclient.NewDefaultClient().LabelNamesInGRPC(ctx, urlParse(t, "http://"+addr), matchers, start, end, limit) if err != nil { return err } @@ -1582,13 +1633,13 @@ func labelNames(t *testing.T, ctx context.Context, addr string, matchers []*labe } //nolint:unparam -func labelValues(t *testing.T, ctx context.Context, addr, label string, matchers []*labels.Matcher, start, end int64, check func(res []string) bool) { +func labelValues(t *testing.T, ctx context.Context, addr, label string, matchers []*labels.Matcher, start, end int64, limit int, check func(res []string) bool) { t.Helper() logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) testutil.Ok(t, runutil.RetryWithLog(logger, 2*time.Second, ctx.Done(), func() error { - res, err := promclient.NewDefaultClient().LabelValuesInGRPC(ctx, urlParse(t, "http://"+addr), label, matchers, start, end) + res, err := promclient.NewDefaultClient().LabelValuesInGRPC(ctx, urlParse(t, "http://"+addr), label, matchers, start, end, limit) if err != nil { return err } @@ -1600,13 +1651,13 @@ func labelValues(t *testing.T, ctx context.Context, addr, label string, matchers })) } -func series(t *testing.T, ctx context.Context, addr string, matchers []*labels.Matcher, start, end int64, check func(res []map[string]string) bool) { +func series(t *testing.T, ctx context.Context, addr string, matchers []*labels.Matcher, start, end int64, limit int, check func(res []map[string]string) bool) { t.Helper() logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) testutil.Ok(t, runutil.RetryWithLog(logger, 2*time.Second, ctx.Done(), func() error { - res, err := promclient.NewDefaultClient().SeriesInGRPC(ctx, urlParse(t, "http://"+addr), matchers, start, end) + res, err := promclient.NewDefaultClient().SeriesInGRPC(ctx, urlParse(t, "http://"+addr), matchers, start, end, limit) if err != nil { return err } @@ -2361,23 +2412,23 @@ func TestQueryTenancyEnforcement(t *testing.T) { }) // default-tenant should only see two labels when enforcing is on (c,tenant_id) - labelNames(t, ctx, querierEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, querierEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 2 }) // default-tenant should only see all labels when enforcing is not on (a,b,c,tenant_id) - labelNames(t, ctx, querierNoEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, querierNoEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 4 }) // default tenant can just the value of the C label labelValues(t, ctx, querierEnforce.Endpoint("http"), "c", nil, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 1 }, ) labelValues(t, ctx, querierEnforce.Endpoint("http"), "a", nil, - timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []string) bool { return len(res) == 0 }, ) @@ -2400,7 +2451,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { matcherSetB = append(matcherSetB, labelMatcher) // default-tenant can see series with matcher C - series(t, ctx, querierEnforce.Endpoint("http"), matcherSetC, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetC, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []map[string]string) bool { var expected = []map[string]string{ { "c": "3", @@ -2411,12 +2462,12 @@ func TestQueryTenancyEnforcement(t *testing.T) { }) // default-tenant cannot see series with matcher B when tenancy is enabled - series(t, ctx, querierEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []map[string]string) bool { return len(res) == 0 }) // default-tenant can see series with matcher B when tenancy is not enabled - series(t, ctx, querierNoEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + series(t, ctx, querierNoEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 0, func(res []map[string]string) bool { var expected = []map[string]string{ { "b": "2", diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index bc3442d04fa..912712f0a8d 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -1388,10 +1388,10 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { u2 := urlParse(t, "http://"+q2.Endpoint("http")) matcher := labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, metricName) // Wait until series can be queried. - series(t, ctx, q1.Endpoint("http"), []*labels.Matcher{matcher}, startMs, endMs, func(res []map[string]string) bool { + series(t, ctx, q1.Endpoint("http"), []*labels.Matcher{matcher}, startMs, endMs, 0, func(res []map[string]string) bool { return len(res) > 0 }) - series(t, ctx, q2.Endpoint("http"), []*labels.Matcher{matcher}, startMs, endMs, func(res []map[string]string) bool { + series(t, ctx, q2.Endpoint("http"), []*labels.Matcher{matcher}, startMs, endMs, 0, func(res []map[string]string) bool { return len(res) > 0 }) @@ -1401,17 +1401,17 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { minT := e2eutil.RandRange(rnd, startMs, endMs) maxT := e2eutil.RandRange(rnd, minT+1, endMs) - res1, err := client.SeriesInGRPC(ctx, u1, matchers, minT, maxT) + res1, err := client.SeriesInGRPC(ctx, u1, matchers, minT, maxT, 0) testutil.Ok(t, err) - res2, err := client.SeriesInGRPC(ctx, u2, matchers, minT, maxT) + res2, err := client.SeriesInGRPC(ctx, u2, matchers, minT, maxT, 0) testutil.Ok(t, err) // Try again with a different timestamp and let requests hit posting cache. minT = e2eutil.RandRange(rnd, startMs, endMs) maxT = e2eutil.RandRange(rnd, minT+1, endMs) - newRes1, err := client.SeriesInGRPC(ctx, u1, matchers, minT, maxT) + newRes1, err := client.SeriesInGRPC(ctx, u1, matchers, minT, maxT, 0) testutil.Ok(t, err) - newRes2, err := client.SeriesInGRPC(ctx, u2, matchers, minT, maxT) + newRes2, err := client.SeriesInGRPC(ctx, u2, matchers, minT, maxT, 0) testutil.Ok(t, err) cases = append(cases, &testCase{