From 0e7136dc0d352c963c55d53f048d0221d389acc6 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Mon, 16 Mar 2020 11:33:25 -0400
Subject: [PATCH 1/2] Add a test for ensuring stats computed from memchunk are
 correct.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 pkg/chunkenc/memchunk_test.go | 68 +++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/pkg/chunkenc/memchunk_test.go b/pkg/chunkenc/memchunk_test.go
index 0d47e91943730..073d138098165 100644
--- a/pkg/chunkenc/memchunk_test.go
+++ b/pkg/chunkenc/memchunk_test.go
@@ -3,6 +3,7 @@ package chunkenc
 import (
 	"bytes"
 	"context"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/rand"
@@ -18,6 +19,7 @@ import (
 	"github.com/grafana/loki/pkg/iter"
 	"github.com/grafana/loki/pkg/logproto"
 	"github.com/grafana/loki/pkg/logql"
+	"github.com/grafana/loki/pkg/logql/stats"
 )
 
 var testEncoding = []Encoding{
@@ -397,6 +399,72 @@ func TestChunkSize(t *testing.T) {
 	}
 }
 
+func TestChunkStats(t *testing.T) {
+	c := NewMemChunk(EncSnappy)
+	first := time.Now()
+	entry := &logproto.Entry{
+		Timestamp: first,
+		Line:      `ts=2020-03-16T13:58:33.459Z caller=dedupe.go:112 component=remote level=debug remote_name=3ea44a url=https:/blan.goo.net/api/prom/push msg=QueueManager.updateShardsLoop lowerBound=45.5 desiredShards=56.724401194003136 upperBound=84.5`,
+	}
+	inserted := 0
+	// fill the chunk with known data size.
+	for {
+		if !c.SpaceFor(entry) {
+			break
+		}
+		if err := c.Append(entry); err != nil {
+			t.Fatal(err)
+		}
+		inserted++
+		entry.Timestamp = entry.Timestamp.Add(time.Nanosecond)
+	}
+	expectedSize := (inserted * len(entry.Line)) + (inserted * 2 * binary.MaxVarintLen64)
+	ctx := stats.NewContext(context.Background())
+
+	it, err := c.Iterator(ctx, first.Add(-time.Hour), entry.Timestamp.Add(time.Hour), logproto.BACKWARD, logql.LineFilterFunc(func(line []byte) bool { return false }))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for it.Next() {
+
+	}
+	if err := it.Close(); err != nil {
+		t.Fatal(err)
+	}
+	s := stats.Snapshot(ctx, time.Since(first))
+	require.Equal(t, int64(expectedSize), s.Summary.TotalBytesProcessed)
+	require.Equal(t, int64(inserted), s.Summary.TotalLinesProcessed)
+
+	require.Equal(t, int64(expectedSize), s.Store.DecompressedBytes)
+	require.Equal(t, int64(inserted), s.Store.DecompressedLines)
+
+	b, err := c.Bytes()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cb, err := NewByteChunk(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx = stats.NewContext(context.Background())
+	it, err = cb.Iterator(ctx, first.Add(-time.Hour), entry.Timestamp.Add(time.Hour), logproto.BACKWARD, logql.LineFilterFunc(func(line []byte) bool { return false }))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for it.Next() {
+
+	}
+	if err := it.Close(); err != nil {
+		t.Fatal(err)
+	}
+	s = stats.Snapshot(ctx, time.Since(first))
+	require.Equal(t, int64(expectedSize), s.Summary.TotalBytesProcessed)
+	require.Equal(t, int64(inserted), s.Summary.TotalLinesProcessed)
+
+	require.Equal(t, int64(expectedSize), s.Store.DecompressedBytes)
+	require.Equal(t, int64(inserted), s.Store.DecompressedLines)
+}
+
 func TestIteratorClose(t *testing.T) {
 	for _, enc := range testEncoding {
 		t.Run(enc.String(), func(t *testing.T) {

From 57f360e152df82d2ad8d7f678a00cd99c502c192 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Mon, 16 Mar 2020 12:29:43 -0400
Subject: [PATCH 2/2] Add some comment

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 pkg/chunkenc/memchunk_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/chunkenc/memchunk_test.go b/pkg/chunkenc/memchunk_test.go
index 073d138098165..d3bccd97d349e 100644
--- a/pkg/chunkenc/memchunk_test.go
+++ b/pkg/chunkenc/memchunk_test.go
@@ -431,6 +431,7 @@ func TestChunkStats(t *testing.T) {
 	if err := it.Close(); err != nil {
 		t.Fatal(err)
 	}
+	// test on a chunk filling up
 	s := stats.Snapshot(ctx, time.Since(first))
 	require.Equal(t, int64(expectedSize), s.Summary.TotalBytesProcessed)
 	require.Equal(t, int64(inserted), s.Summary.TotalLinesProcessed)
@@ -442,6 +443,8 @@ func TestChunkStats(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// test on a new chunk.
 	cb, err := NewByteChunk(b)
 	if err != nil {
 		t.Fatal(err)