Skip to content
This repository has been archived by the owner on Aug 13, 2019. It is now read-only.

add function to retreive index's section memory size and use this to get symbol table size #272

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion block.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ type IndexReader interface {
// LabelIndices returns a list of string tuples for which a label value index exists.
LabelIndices() ([][]string, error)

// GetSymbolTableSize returns the size occupied by the symbol table of Reader object.
GetSymbolTableSize() uint32

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This has nothing to do with functionality of an IndexReader and makes strong assumptions about how it is implemented. However we want to expose it, this interface isn't it.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had the same question in this #297 PR.
There I have added a new interface IndexExtra for exposing LabelName() function.
Does this way work or is there any other way to expose functions from index.go ?

// Close releases the underlying resources of the reader.
Close() error
}
Expand Down Expand Up @@ -382,6 +385,11 @@ func (r blockIndexReader) LabelIndices() ([][]string, error) {
return ss, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
}

func (r blockIndexReader) GetSymbolTableSize() uint32 {
ss := r.ir.GetSymbolTableSize()
return ss
}

func (r blockIndexReader) Close() error {
r.b.pendingReaders.Done()
return nil
Expand Down Expand Up @@ -474,7 +482,7 @@ func (pb *Block) CleanTombstones(dest string, c Compactor) (bool, error) {
numStones := 0

pb.tombstones.Iter(func(id uint64, ivs Intervals) error {
for _ = range ivs {
for range ivs {
numStones++
}

Expand Down
20 changes: 19 additions & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ type dbMetrics struct {
cutoffs prometheus.Counter
cutoffsFailed prometheus.Counter
tombCleanTimer prometheus.Histogram
symbolTableSize prometheus.GaugeFunc
}

func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {
Expand Down Expand Up @@ -162,7 +163,23 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {
Name: "prometheus_tsdb_tombstone_cleanup_seconds",
Help: "The time taken to recompact blocks to remove tombstones.",
})

m.symbolTableSize = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "prometheus_tsdb_symbol_table_size",
Help: "Size occupied by symbol table",
}, func() float64 {
db.mtx.RLock()
defer db.mtx.RUnlock()
var size uint64
for _, block := range db.blocks {
ind, err := block.Index()
if err != nil {
size = 0
break
}
size += uint64(ind.GetSymbolTableSize())
}
return float64(size)
})
if r != nil {
r.MustRegister(
m.loadedBlocks,
Expand All @@ -172,6 +189,7 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {
m.cutoffsFailed,
m.compactionsTriggered,
m.tombCleanTimer,
m.symbolTableSize,
)
}
return m
Expand Down
17 changes: 17 additions & 0 deletions index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -749,6 +749,23 @@ func (r *Reader) decbufUvarintAt(off int) decbuf {
return dec
}

// getSectionSize returns the first 4 bytes in a section starting at offset off
// to get the content length bytes.
func (r *Reader) getSectionSize(off int) uint32 {
b := r.b.Range(off, off+4)
l := uint32(binary.BigEndian.Uint32(b))

if l <= 0 {
l = 0
}
return l
}

// getSymbolTableSize returns the bytes taken by the symbol table of Reader object.
func (r *Reader) GetSymbolTableSize() uint32 {
return r.getSectionSize(int(r.toc.symbols))
}

// readSymbols reads the symbol table fully into memory and allocates proper strings for them.
// Strings backed by the mmap'd memory would cause memory faults if applications keep using them
// after the reader is closed.
Expand Down