From b8aace249e31fe1248c79b04dfcfb1b7222a0ad5 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Fri, 25 Aug 2023 02:56:46 -0600 Subject: [PATCH] [release-2.9.x] Series Index Store: fix race in GetSeries (#10346) Backport cf353bb9a0ae8452d7fc2af409a3f6eeaaf099c8 from #10310 --- **What this PR does / why we need it**: ~Lock around data that is modified from multiple goroutines concurrently.~ Replaced with the implementation from #9984 since @akhilanarayanan did it more efficiently. **Which issue(s) this PR fixes**: Relates to #8586 **Checklist** - [x] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - NA Documentation added - NA Tests updated - [x] `CHANGELOG.md` updated - NA If the change is worth mentioning in the release notes, add `add-to-release-notes` label - NA Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - NA For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213) Co-authored-by: Bryan Boreham --- CHANGELOG.md | 1 + pkg/storage/stores/series/series_index_store.go | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cee5335f7601d..47232dd9bee79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ * [9773](https://github.com/grafana/loki/pull/9773) **ssncferreira**: Fix instant query summary statistic's `splits` corresponding to the number of subqueries a query is split into based on `split_queries_by_interval`. * [9949](https://github.com/grafana/loki/pull/9949) **masslessparticle**: Fix pipelines to clear caches when tailing to avoid resource exhaustion. * [9936](https://github.com/grafana/loki/pull/9936) **masslessparticle**: Fix the way query stages are reordered when `unpack` is present. +* [10309](https://github.com/grafana/loki/pull/10309) **akhilanarayanan**: Fix race condition in series index store. ##### Changes diff --git a/pkg/storage/stores/series/series_index_store.go b/pkg/storage/stores/series/series_index_store.go index da92fdbf60c9b..1b445572bd99c 100644 --- a/pkg/storage/stores/series/series_index_store.go +++ b/pkg/storage/stores/series/series_index_store.go @@ -279,7 +279,7 @@ func (c *indexReaderWriter) chunksToSeries(ctx context.Context, in []logproto.Ch })) } - results := make([]labels.Labels, 0, len(chunksBySeries)) + perJobResults := make([][]labels.Labels, len(jobs)) // Picking an arbitrary bound of 20 numConcurrent jobs. numConcurrent := len(jobs) @@ -294,7 +294,7 @@ func (c *indexReaderWriter) chunksToSeries(ctx context.Context, in []logproto.Ch func(_ context.Context, idx int) error { res, err := jobs[idx]() if res != nil { - results = append(results, res...) + perJobResults[idx] = res } return err }, @@ -302,6 +302,10 @@ func (c *indexReaderWriter) chunksToSeries(ctx context.Context, in []logproto.Ch return nil, err } + results := make([]labels.Labels, len(chunksBySeries)) // Flatten out the per-job results. + for _, innerSlice := range perJobResults { + results = append(results, innerSlice...) + } sort.Slice(results, func(i, j int) bool { return labels.Compare(results[i], results[j]) < 0 })