From fd563298d92b4e89cf027e811f50aaed737dca24 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 2 Oct 2019 15:30:52 -0400 Subject: [PATCH 01/13] Fixed two series already published by loki to use the cortex names Signed-off-by: Joe Elliott --- production/loki-mixin/dashboards.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index dce926923763f..a2f60c2902263 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -116,14 +116,14 @@ local utils = import "mixin-utils/utils.libsonnet"; ) .addPanel( g.panel('Entries') + - g.queryPanel('sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster="$cluster", job="$namespace/ingester"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster="$cluster", job="$namespace/ingester"}[5m]))', 'entries'), + g.queryPanel('sum(rate(cortex_chunk_store_index_entries_per_chunk_sum{cluster="$cluster", job="$namespace/ingester"}[5m])) / sum(rate(cortex_chunk_store_index_entries_per_chunk_count{cluster="$cluster", job="$namespace/ingester"}[5m]))', 'entries'), ), ) .addRow( g.row('Flush Stats') .addPanel( g.panel('Queue Length') + - g.queryPanel('loki_ingester_flush_queue_length{cluster="$cluster", job="$namespace/ingester"}', '{{instance}}'), + g.queryPanel('cortex_ingester_flush_queue_length{cluster="$cluster", job="$namespace/ingester"}', '{{instance}}'), ) .addPanel( g.panel('Flush Rate') + From ec02384f61f72470e09927629619a0c46f43bfda Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 2 Oct 2019 16:45:23 -0400 Subject: [PATCH 02/13] Support loki_ingester_memory_chunks Signed-off-by: Joe Elliott --- pkg/ingester/flush.go | 6 ++++++ pkg/ingester/instance.go | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index d45c11bea6377..13108605eb27c 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -22,6 +22,10 @@ import ( ) var ( + memoryChunks = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "loki_ingester_memory_chunks", + Help: "The total number of chunks in memory.", + }) chunkEntries = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "loki_ingester_chunk_entries", Help: "Distribution of stored chunk entries (when stored).", @@ -234,6 +238,7 @@ func (i *Ingester) shouldFlushChunk(chunk *chunkDesc) bool { func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) { now := time.Now() + prevNumChunks := len(stream.chunks) for len(stream.chunks) > 0 { if stream.chunks[0].flushed.IsZero() || now.Sub(stream.chunks[0].flushed) < i.cfg.RetainPeriod { break @@ -242,6 +247,7 @@ func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) { stream.chunks[0].chunk = nil // erase reference so the chunk can be garbage-collected stream.chunks = stream.chunks[1:] } + memoryChunks.Sub(float64(prevNumChunks - len(stream.chunks))) if len(stream.chunks) == 0 { delete(instance.streams, stream.fp) diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 18c386bc379f4..0c677c9d3a024 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -92,7 +92,12 @@ func (i *instance) consumeChunk(ctx context.Context, labels []client.LabelAdapte i.addTailersToNewStream(stream) } - return stream.consumeChunk(ctx, chunk) + err := stream.consumeChunk(ctx, chunk) + if err != nil { + memoryChunks.Inc() + } + + return err } func (i *instance) Push(ctx context.Context, req *logproto.PushRequest) error { @@ -113,10 +118,13 @@ func (i *instance) Push(ctx context.Context, req *logproto.PushRequest) error { continue } + prevNumChunks := len(stream.chunks) if err := stream.Push(ctx, s.Entries); err != nil { appendErr = err continue } + + memoryChunks.Add(float64(len(stream.chunks) - prevNumChunks)) } return appendErr From d5f52d2a2d5ef31c980697e837790e7b95728a3c Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 2 Oct 2019 17:06:06 -0400 Subject: [PATCH 03/13] Added loki_ingester_memory_streams Signed-off-by: Joe Elliott --- pkg/ingester/flush.go | 1 + pkg/ingester/instance.go | 5 +++++ production/loki-mixin/dashboards.libsonnet | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index 13108605eb27c..9c44a2f8b7dbd 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -253,6 +253,7 @@ func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) { delete(instance.streams, stream.fp) instance.index.Delete(client.FromLabelAdaptersToLabels(stream.labels), stream.fp) instance.streamsRemovedTotal.Inc() + memoryStreams.Dec() } } diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 0c677c9d3a024..faa0d1cea9c63 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -32,6 +32,10 @@ var ( ) var ( + memoryStreams = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "loki_ingester_memory_streams", + Help: "The total number of streams in memory.", + }) streamsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_created_total", @@ -89,6 +93,7 @@ func (i *instance) consumeChunk(ctx context.Context, labels []client.LabelAdapte i.index.Add(labels, fp) i.streams[fp] = stream i.streamsCreatedTotal.Inc() + memoryStreams.Inc() i.addTailersToNewStream(stream) } diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index a2f60c2902263..60d58e2e098cb 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -92,7 +92,7 @@ local utils = import "mixin-utils/utils.libsonnet"; ) .addPanel( g.panel('Chunks per series') + - g.queryPanel('sum(loki_ingester_memory_chunks{cluster="$cluster", job="$namespace/ingester"}) / sum(loki_ingester_memory_series{job="$namespace/ingester"})', 'chunks'), + g.queryPanel('sum(loki_ingester_memory_chunks{cluster="$cluster", job="$namespace/ingester"}) / sum(loki_ingester_memory_streams{job="$namespace/ingester"})', 'chunks'), ) ) .addRow( From 26b7395ffc523f73e23d28c58a432564edf09d93 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 2 Oct 2019 17:14:00 -0400 Subject: [PATCH 04/13] Added increment during normal stream creation Signed-off-by: Joe Elliott --- pkg/ingester/instance.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index faa0d1cea9c63..c3eaf57a95aed 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -149,6 +149,7 @@ func (i *instance) getOrCreateStream(labels []client.LabelAdapter) (*stream, err stream = newStream(fp, labels, i.blockSize) i.index.Add(labels, fp) i.streams[fp] = stream + memoryStreams.Inc() i.streamsCreatedTotal.Inc() i.addTailersToNewStream(stream) From 54700fa83d9d4cedbcc4e25bf6446ea21fa902f3 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 11:46:02 -0400 Subject: [PATCH 05/13] Added Support for Utilization metrics at the Chunk level Signed-off-by: Joe Elliott --- pkg/chunkenc/dumb_chunk.go | 5 +++++ pkg/chunkenc/facade.go | 4 ++++ pkg/chunkenc/gzip.go | 7 +++++++ pkg/chunkenc/interface.go | 1 + pkg/ingester/flush.go | 6 ++++++ 5 files changed, 23 insertions(+) diff --git a/pkg/chunkenc/dumb_chunk.go b/pkg/chunkenc/dumb_chunk.go index 89a3b4128feef..c4e507a4e03de 100644 --- a/pkg/chunkenc/dumb_chunk.go +++ b/pkg/chunkenc/dumb_chunk.go @@ -50,6 +50,11 @@ func (c *dumbChunk) Size() int { return len(c.entries) } +// Utilization implements Chunk +func (c *dumbChunk) Utilization() float64 { + return float64(len(c.entries)) / float64(tmpNumEntries) +} + // Returns an iterator that goes from _most_ recent to _least_ recent (ie, // backwards). func (c *dumbChunk) Iterator(from, through time.Time, direction logproto.Direction, _ logql.Filter) (iter.EntryIterator, error) { diff --git a/pkg/chunkenc/facade.go b/pkg/chunkenc/facade.go index ef3801d1dc222..658d4af156e7b 100644 --- a/pkg/chunkenc/facade.go +++ b/pkg/chunkenc/facade.go @@ -56,3 +56,7 @@ func (Facade) Encoding() encoding.Encoding { func (f Facade) LokiChunk() Chunk { return f.c } + +func (f Facade) Utilization() float64 { + return f.c.Utilization() +} diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 3d42dd5b5189b..ac4a68ef2fba1 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -313,6 +313,13 @@ func (c *MemChunk) SpaceFor(*logproto.Entry) bool { return len(c.blocks) < blocksPerChunk } +// Utilization implements Chunk. +func (c *MemChunk) Utilization() float64 { + // Here we're just reporting the block utilization which is not super exciting. It would be more interesting to see + // for each block what %age of c.blocksize was used, but that would require uncompressing the data in every block + return float64(len(c.blocks)) / float64(blocksPerChunk) +} + // Append implements Chunk. func (c *MemChunk) Append(entry *logproto.Entry) error { entryTimestamp := entry.Timestamp.UnixNano() diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index 08ee8cf3034c2..b574f765087cd 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -50,6 +50,7 @@ type Chunk interface { Iterator(from, through time.Time, direction logproto.Direction, filter logql.Filter) (iter.EntryIterator, error) Size() int Bytes() ([]byte, error) + Utilization() float64 } // CompressionWriter is the writer that compresses the data passed to it. diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index 9c44a2f8b7dbd..44aa2cfa07c65 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -22,6 +22,11 @@ import ( ) var ( + chunkUtilization = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "loki_ingester_chunk_utilization", + Help: "Distribution of stored chunk utilization (when stored).", + Buckets: prometheus.LinearBuckets(0, 0.2, 6), + }) memoryChunks = promauto.NewGauge(prometheus.GaugeOpts{ Name: "loki_ingester_memory_chunks", Help: "The total number of chunks in memory.", @@ -299,6 +304,7 @@ func (i *Ingester) flushChunks(ctx context.Context, fp model.Fingerprint, labelP continue } + chunkUtilization.Observe(wc.Data.Utilization()) chunkEntries.Observe(float64(numEntries)) chunkSize.Observe(float64(len(byt))) sizePerTenant.Add(float64(len(byt))) From 2fa25a8fa57a63937c31891e2756c9e39aa2f745 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 12:12:52 -0400 Subject: [PATCH 06/13] Used loki_ingester_chunk_entries instead of length --- production/loki-mixin/dashboards.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index 60d58e2e098cb..4129bdd25f472 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -111,7 +111,7 @@ local utils = import "mixin-utils/utils.libsonnet"; g.row('Flush Stats') .addPanel( g.panel('Size') + - g.latencyPanel('loki_ingester_chunk_length', '{cluster="$cluster", job="$namespace/ingester"}', multiplier='1') + + g.latencyPanel('loki_ingester_chunk_entries', '{cluster="$cluster", job="$namespace/ingester"}', multiplier='1') + { yaxes: g.yaxes('short') }, ) .addPanel( From 918db8ec6b8176e5bc3a761e22e3ae923789b419 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 13:18:17 -0400 Subject: [PATCH 07/13] Added/renamed metrics in docs Signed-off-by: Joe Elliott --- docs/operations/observability.md | 36 ++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/operations/observability.md b/docs/operations/observability.md index fcbe10f7346c3..c33a34fad22c8 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -24,22 +24,26 @@ The Loki Distributors expose the following metrics: The Loki Ingesters expose the following metrics: -| Metric Name | Metric Type | Description | -| ----------------------------------------- | ----------- | ------------------------------------------------------------------------------------------- | -| `cortex_ingester_flush_queue_length` | Gauge | The total number of series pending in the flush queue. | -| `loki_ingester_chunk_age_seconds` | Histogram | Distribution of chunk ages when flushed. | -| `loki_ingester_chunk_encode_time_seconds` | Histogram | Distribution of chunk encode times. | -| `loki_ingester_chunk_entries` | Histogram | Distribution of entires per-chunk when flushed. | -| `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | -| `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | -| `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. | -| `loki_ingester_chunks_flushed_total` | Counter | The total number of chunks flushed by the ingester. | -| `loki_ingester_chunks_stored_total` | Counter | Total stored chunks per tenant. | -| `loki_ingester_received_chunks` | Counter | The total number of chunks sent by this ingester whilst joining during the handoff process. | -| `loki_ingester_samples_per_chunk` | Histogram | The number of samples in a chunk. | -| `loki_ingester_sent_chunks` | Counter | The total number of chunks sent by this ingester whilst leaving during the handoff process. | -| `loki_ingester_streams_created_total` | Counter | The total number of streams created per tenant. | -| `loki_ingester_streams_removed_total` | Counter | The total number of streams removed per tenant. | +| Metric Name | Metric Type | Description | +| -------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------- | +| `cortex_ingester_flush_queue_length` | Gauge | The total number of series pending in the flush queue. | +| `cortex_chunk_store_index_entries_per_chunk` | Histogram | Number of index entries written to storage per chunk. | +| `loki_ingester_memory_chunks` | Gauge | The total number of chunks in memory. | +| `loki_ingester_memory_streams` | Gauge | The total number of streams in memory. | +| `loki_ingester_chunk_age_seconds` | Histogram | Distribution of chunk ages when flushed. | +| `loki_ingester_chunk_encode_time_seconds` | Histogram | Distribution of chunk encode times. | +| `loki_ingester_chunk_entries` | Histogram | Distribution of entries per-chunk when flushed. | +| `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | +| `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization when flushed. | +| `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | +| `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. | +| `loki_ingester_chunks_flushed_total` | Counter | The total number of chunks flushed by the ingester. | +| `loki_ingester_chunks_stored_total` | Counter | Total stored chunks per tenant. | +| `loki_ingester_received_chunks` | Counter | The total number of chunks sent by this ingester whilst joining during the handoff process. | +| `loki_ingester_samples_per_chunk` | Histogram | The number of samples in a chunk. | +| `loki_ingester_sent_chunks` | Counter | The total number of chunks sent by this ingester whilst leaving during the handoff process. | +| `loki_ingester_streams_created_total` | Counter | The total number of streams created per tenant. | +| `loki_ingester_streams_removed_total` | Counter | The total number of streams removed per tenant. | Promtail exposes these metrics: From 66d36a01404a0abdd921ba79d17d1bf05cd93265 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 13:21:49 -0400 Subject: [PATCH 08/13] Fixed memory chunk increment on no error Signed-off-by: Joe Elliott --- pkg/ingester/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index c3eaf57a95aed..a10fe6e9a4e29 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -98,7 +98,7 @@ func (i *instance) consumeChunk(ctx context.Context, labels []client.LabelAdapte } err := stream.consumeChunk(ctx, chunk) - if err != nil { + if err == nil { memoryChunks.Inc() } From 87a207b66a1a3178bb6d16a129caa44577748c4b Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 14:54:02 -0400 Subject: [PATCH 09/13] Updated _entries description to be more meaningful Signed-off-by: Joe Elliott --- docs/operations/observability.md | 2 +- pkg/ingester/flush.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations/observability.md b/docs/operations/observability.md index c33a34fad22c8..821306b3c61ad 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -32,7 +32,7 @@ The Loki Ingesters expose the following metrics: | `loki_ingester_memory_streams` | Gauge | The total number of streams in memory. | | `loki_ingester_chunk_age_seconds` | Histogram | Distribution of chunk ages when flushed. | | `loki_ingester_chunk_encode_time_seconds` | Histogram | Distribution of chunk encode times. | -| `loki_ingester_chunk_entries` | Histogram | Distribution of entries per-chunk when flushed. | +| `loki_ingester_chunk_entries` | Histogram | Distribution of lines per-chunk when flushed. | | `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | | `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization when flushed. | | `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index 44aa2cfa07c65..bd738c35e0a3d 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -33,7 +33,7 @@ var ( }) chunkEntries = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "loki_ingester_chunk_entries", - Help: "Distribution of stored chunk entries (when stored).", + Help: "Distribution of stored lines per chunk (when stored).", Buckets: prometheus.ExponentialBuckets(200, 2, 9), // biggest bucket is 200*2^(9-1) = 51200 }) chunkSize = promauto.NewHistogram(prometheus.HistogramOpts{ From b4a191a822db56084d6dc3743547bd6fb35aae43 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 7 Oct 2019 21:22:15 -0400 Subject: [PATCH 10/13] Changed utilization metric to be used uncompressed bytes / total uncompressed bytes Signed-off-by: Joe Elliott --- pkg/chunkenc/gzip.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index ac4a68ef2fba1..9fb6c520a17a4 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -61,7 +61,8 @@ type block struct { mint, maxt int64 - offset int // The offset of the block in the chunk. + offset int // The offset of the block in the chunk. + uncompressedSize int // Total uncompressed size in bytes when the chunk is cut. } // This block holds the un-compressed entries. Once it has enough data, this is @@ -313,11 +314,19 @@ func (c *MemChunk) SpaceFor(*logproto.Entry) bool { return len(c.blocks) < blocksPerChunk } -// Utilization implements Chunk. +// Utilization implements Chunk. It is the bytes used as a percentage of the func (c *MemChunk) Utilization() float64 { - // Here we're just reporting the block utilization which is not super exciting. It would be more interesting to see - // for each block what %age of c.blocksize was used, but that would require uncompressing the data in every block - return float64(len(c.blocks)) / float64(blocksPerChunk) + usedSize := 0 + + if !c.head.isEmpty() { + usedSize += c.head.size + } + + for _, b := range c.blocks { + usedSize += b.uncompressedSize + } + + return float64(usedSize) / float64(blocksPerChunk*c.blockSize) } // Append implements Chunk. @@ -359,10 +368,11 @@ func (c *MemChunk) cut() error { } c.blocks = append(c.blocks, block{ - b: b, - numEntries: len(c.head.entries), - mint: c.head.mint, - maxt: c.head.maxt, + b: b, + numEntries: len(c.head.entries), + mint: c.head.mint, + maxt: c.head.maxt, + uncompressedSize: c.head.size, }) c.head.entries = c.head.entries[:0] From 15483a61e5b5f8ab0eee440fa0a04a9be506df1c Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 8 Oct 2019 09:40:46 -0400 Subject: [PATCH 11/13] Added loki_ingester_chunk_compression_ratio Signed-off-by: Joe Elliott --- docs/operations/observability.md | 1 + pkg/chunkenc/dumb_chunk.go | 5 +++++ pkg/chunkenc/facade.go | 16 ++++++++++++++-- pkg/chunkenc/gzip.go | 19 +++++++++++++------ pkg/chunkenc/interface.go | 1 + pkg/ingester/flush.go | 16 ++++++++++++++-- 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 821306b3c61ad..9420d4706fd6a 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -35,6 +35,7 @@ The Loki Ingesters expose the following metrics: | `loki_ingester_chunk_entries` | Histogram | Distribution of lines per-chunk when flushed. | | `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | | `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization when flushed. | +| `loki_ingester_chunk_compression_ratio` | Histogram | Distribution of chunk compression ratio when flushed. | | `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | | `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. | | `loki_ingester_chunks_flushed_total` | Counter | The total number of chunks flushed by the ingester. | diff --git a/pkg/chunkenc/dumb_chunk.go b/pkg/chunkenc/dumb_chunk.go index c4e507a4e03de..edb154b050b8e 100644 --- a/pkg/chunkenc/dumb_chunk.go +++ b/pkg/chunkenc/dumb_chunk.go @@ -50,6 +50,11 @@ func (c *dumbChunk) Size() int { return len(c.entries) } +// UncompressedSize implements Chunk. +func (c *dumbChunk) UncompressedSize() int { + return c.Size() +} + // Utilization implements Chunk func (c *dumbChunk) Utilization() float64 { return float64(len(c.entries)) / float64(tmpNumEntries) diff --git a/pkg/chunkenc/facade.go b/pkg/chunkenc/facade.go index 658d4af156e7b..8556b0fdd0442 100644 --- a/pkg/chunkenc/facade.go +++ b/pkg/chunkenc/facade.go @@ -52,11 +52,23 @@ func (Facade) Encoding() encoding.Encoding { return GzipLogChunk } +// Utilization implements encoding.Chunk. +func (f Facade) Utilization() float64 { + return f.c.Utilization() +} + // LokiChunk returns the chunkenc.Chunk. func (f Facade) LokiChunk() Chunk { return f.c } -func (f Facade) Utilization() float64 { - return f.c.Utilization() +// UncompressedSize is a helper function to hide the type assertion kludge when wanting the uncompressed size of the Cortex interface encoding.Chunk. +func UncompressedSize(c encoding.Chunk) (int, bool) { + f, ok := c.(*Facade) + + if !ok { + return 0, false + } + + return f.c.UncompressedSize(), true } diff --git a/pkg/chunkenc/gzip.go b/pkg/chunkenc/gzip.go index 9fb6c520a17a4..075373b40c9fd 100644 --- a/pkg/chunkenc/gzip.go +++ b/pkg/chunkenc/gzip.go @@ -314,19 +314,26 @@ func (c *MemChunk) SpaceFor(*logproto.Entry) bool { return len(c.blocks) < blocksPerChunk } -// Utilization implements Chunk. It is the bytes used as a percentage of the -func (c *MemChunk) Utilization() float64 { - usedSize := 0 +// UncompressedSize implements Chunk. +func (c *MemChunk) UncompressedSize() int { + size := 0 if !c.head.isEmpty() { - usedSize += c.head.size + size += c.head.size } for _, b := range c.blocks { - usedSize += b.uncompressedSize + size += b.uncompressedSize } - return float64(usedSize) / float64(blocksPerChunk*c.blockSize) + return size +} + +// Utilization implements Chunk. It is the bytes used as a percentage of the +func (c *MemChunk) Utilization() float64 { + size := c.UncompressedSize() + + return float64(size) / float64(blocksPerChunk*c.blockSize) } // Append implements Chunk. diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index b574f765087cd..1f8d405e8b4b3 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -51,6 +51,7 @@ type Chunk interface { Size() int Bytes() ([]byte, error) Utilization() float64 + UncompressedSize() int } // CompressionWriter is the writer that compresses the data passed to it. diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index bd738c35e0a3d..29f45a9a28133 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -41,6 +41,11 @@ var ( Help: "Distribution of stored chunk sizes (when stored).", Buckets: prometheus.ExponentialBuckets(10000, 2, 7), // biggest bucket is 10000*2^(7-1) = 640000 (~640KB) }) + chunkCompressionRatio = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "loki_ingester_chunk_compression_ratio", + Help: "Compression ratio of chunks (when stored).", + Buckets: prometheus.LinearBuckets(1, 1.5, 6), + }) chunksPerTenant = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "loki_ingester_chunks_stored_total", Help: "Total stored chunks per tenant.", @@ -304,10 +309,17 @@ func (i *Ingester) flushChunks(ctx context.Context, fp model.Fingerprint, labelP continue } + compressedSize := float64(len(byt)) + uncompressedSize, ok := chunkenc.UncompressedSize(wc.Data) + + if ok { + chunkCompressionRatio.Observe(float64(uncompressedSize) / compressedSize) + } + chunkUtilization.Observe(wc.Data.Utilization()) chunkEntries.Observe(float64(numEntries)) - chunkSize.Observe(float64(len(byt))) - sizePerTenant.Add(float64(len(byt))) + chunkSize.Observe(compressedSize) + sizePerTenant.Add(compressedSize) countPerTenant.Inc() firstTime, _ := cs[i].chunk.Bounds() chunkAge.Observe(time.Since(firstTime).Seconds()) From 55aa74318edfd1d8bc439f8837180b4f894aa1ad Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 8 Oct 2019 10:44:43 -0400 Subject: [PATCH 12/13] Removed unused flushed chunks metric from documentation/code Signed-off-by: Joe Elliott --- docs/operations/observability.md | 1 - pkg/ingester/stream.go | 6 ------ 2 files changed, 7 deletions(-) diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 9420d4706fd6a..60234f33c6455 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -38,7 +38,6 @@ The Loki Ingesters expose the following metrics: | `loki_ingester_chunk_compression_ratio` | Histogram | Distribution of chunk compression ratio when flushed. | | `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | | `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. | -| `loki_ingester_chunks_flushed_total` | Counter | The total number of chunks flushed by the ingester. | | `loki_ingester_chunks_stored_total` | Counter | Total stored chunks per tenant. | | `loki_ingester_received_chunks` | Counter | The total number of chunks sent by this ingester whilst joining during the handoff process. | | `loki_ingester_samples_per_chunk` | Histogram | The number of samples in a chunk. | diff --git a/pkg/ingester/stream.go b/pkg/ingester/stream.go index 9e8f857f3cc30..095f5137cbd96 100644 --- a/pkg/ingester/stream.go +++ b/pkg/ingester/stream.go @@ -25,11 +25,6 @@ var ( Name: "ingester_chunks_created_total", Help: "The total number of chunks created in the ingester.", }) - chunksFlushedTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "loki", - Name: "ingester_chunks_flushed_total", - Help: "The total number of chunks flushed by the ingester.", - }) samplesPerChunk = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "loki", Subsystem: "ingester", @@ -42,7 +37,6 @@ var ( func init() { prometheus.MustRegister(chunksCreatedTotal) - prometheus.MustRegister(chunksFlushedTotal) prometheus.MustRegister(samplesPerChunk) } From 27237f2994a0eba0bc8d09f4ac3198c557c75331 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 8 Oct 2019 12:03:55 -0400 Subject: [PATCH 13/13] Added a more detailed description for utilization Co-Authored-By: Ed Welch --- docs/operations/observability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 60234f33c6455..20548a76ac1de 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -34,7 +34,7 @@ The Loki Ingesters expose the following metrics: | `loki_ingester_chunk_encode_time_seconds` | Histogram | Distribution of chunk encode times. | | `loki_ingester_chunk_entries` | Histogram | Distribution of lines per-chunk when flushed. | | `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | -| `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization when flushed. | +| `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization (filled uncompressed bytes vs maximum uncompressed bytes) when flushed. | | `loki_ingester_chunk_compression_ratio` | Histogram | Distribution of chunk compression ratio when flushed. | | `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | | `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. |