From f46f8eabeff4a7771950a4218ae1c30bd243805b Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Tue, 10 Dec 2019 20:07:16 +0530 Subject: [PATCH] Store total key-value size in table footer (#1137) This PR stores the total key-value size in a table in the table footer. The key-value size can be accessed via db.Tables(..) call. It returns the list of all tables along with the total size of key-values. --- db.go | 7 ++- db2_test.go | 2 +- levels.go | 28 ++++++---- manifest_test.go | 2 +- pb/pb.pb.go | 118 +++++++++++++++++++++++++++--------------- pb/pb.proto | 5 +- stream_writer.go | 6 ++- table/builder.go | 18 ++++--- table/builder_test.go | 4 +- table/table.go | 7 +++ table/table_test.go | 20 +++++-- 11 files changed, 144 insertions(+), 73 deletions(-) diff --git a/db.go b/db.go index dd2f0fb54..4b58fae48 100644 --- a/db.go +++ b/db.go @@ -899,11 +899,16 @@ func buildL0Table(ft flushTask, bopts table.Options) []byte { defer iter.Close() b := table.NewTableBuilder(bopts) defer b.Close() + var vp valuePointer for iter.SeekToFirst(); iter.Valid(); iter.Next() { if len(ft.dropPrefix) > 0 && bytes.HasPrefix(iter.Key(), ft.dropPrefix) { continue } - b.Add(iter.Key(), iter.Value()) + vs := iter.Value() + if vs.Meta&bitValuePointer > 0 { + vp.Decode(vs.Value) + } + b.Add(iter.Key(), iter.Value(), vp.Len) } return b.Finish() } diff --git a/db2_test.go b/db2_test.go index 9362a7219..3885f9aae 100644 --- a/db2_test.go +++ b/db2_test.go @@ -538,7 +538,7 @@ func createTableWithRange(t *testing.T, db *DB, start, end int) *table.Table { binary.BigEndian.PutUint64(key[:], uint64(i)) key = y.KeyWithTs(key, uint64(0)) val := y.ValueStruct{Value: []byte(fmt.Sprintf("%d", i))} - b.Add(key, val) + b.Add(key, val, 0) } fileID := db.lc.reserveFileID() diff --git a/levels.go b/levels.go index 30b753276..0a4b92f26 100644 --- a/levels.go +++ b/levels.go @@ -508,6 +508,7 @@ func (s *levelsController) compactBuildTables( resultCh := make(chan newTableResult) var numBuilds, numVersions int var lastKey, skipKey []byte + var vp valuePointer for it.Valid() { timeStart := time.Now() dk, err := s.kv.registry.latestDataKey() @@ -584,7 +585,10 @@ func (s *levelsController) compactBuildTables( } } numKeys++ - builder.Add(it.Key(), it.Value()) + if vs.Meta&bitValuePointer > 0 { + vp.Decode(vs.Value) + } + builder.Add(it.Key(), vs, vp.Len) } // It was true that it.Valid() at least once in the loop above, which means we // called Add() at least once, and builder is not Empty(). @@ -1009,11 +1013,12 @@ func (s *levelsController) appendIterators( // TableInfo represents the information about a table. type TableInfo struct { - ID uint64 - Level int - Left []byte - Right []byte - KeyCount uint64 // Number of keys in the table + ID uint64 + Level int + Left []byte + Right []byte + KeyCount uint64 // Number of keys in the table + EstimatedSz uint64 } func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo) { @@ -1030,11 +1035,12 @@ func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo) } info := TableInfo{ - ID: t.ID(), - Level: l.level, - Left: t.Smallest(), - Right: t.Biggest(), - KeyCount: count, + ID: t.ID(), + Level: l.level, + Left: t.Smallest(), + Right: t.Biggest(), + KeyCount: count, + EstimatedSz: t.EstimatedSize(), } result = append(result, info) } diff --git a/manifest_test.go b/manifest_test.go index f28270f3b..c9a51fc60 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -153,7 +153,7 @@ func buildTable(t *testing.T, keyValues [][]string, bopts table.Options) *os.Fil Value: []byte(kv[1]), Meta: 'A', UserMeta: 0, - }) + }, 0) } _, err = f.Write(b.Finish()) require.NoError(t, err, "unable to write to file.") diff --git a/pb/pb.pb.go b/pb/pb.pb.go index 8542709e8..bb8518594 100644 --- a/pb/pb.pb.go +++ b/pb/pb.pb.go @@ -447,6 +447,7 @@ func (m *BlockOffset) GetLen() uint32 { type TableIndex struct { Offsets []*BlockOffset `protobuf:"bytes,1,rep,name=offsets,proto3" json:"offsets,omitempty"` BloomFilter []byte `protobuf:"bytes,2,opt,name=bloom_filter,json=bloomFilter,proto3" json:"bloom_filter,omitempty"` + EstimatedSize uint64 `protobuf:"varint,3,opt,name=estimated_size,json=estimatedSize,proto3" json:"estimated_size,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -499,6 +500,13 @@ func (m *TableIndex) GetBloomFilter() []byte { return nil } +func (m *TableIndex) GetEstimatedSize() uint64 { + if m != nil { + return m.EstimatedSize + } + return 0 +} + type Checksum struct { Algo Checksum_Algorithm `protobuf:"varint,1,opt,name=algo,proto3,enum=pb.Checksum_Algorithm" json:"algo,omitempty"` Sum uint64 `protobuf:"varint,2,opt,name=sum,proto3" json:"sum,omitempty"` @@ -642,47 +650,48 @@ func init() { func init() { proto.RegisterFile("pb.proto", fileDescriptor_f80abaa17e25ccc8) } var fileDescriptor_f80abaa17e25ccc8 = []byte{ - // 631 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0xcf, 0x6e, 0xd3, 0x4e, - 0x10, 0xce, 0x3a, 0xa9, 0x93, 0x4c, 0xda, 0x34, 0xbf, 0xd5, 0x8f, 0xca, 0x08, 0x08, 0xc6, 0x08, - 0x29, 0x54, 0x55, 0x0e, 0x2d, 0xe2, 0xc2, 0x29, 0x4d, 0x83, 0x88, 0xd2, 0x2a, 0xd2, 0x52, 0x55, - 0x15, 0x97, 0x68, 0x63, 0x4f, 0x1a, 0xcb, 0x7f, 0xd6, 0xb2, 0x37, 0x56, 0xf3, 0x26, 0x3c, 0x12, - 0x47, 0x0e, 0x3c, 0x00, 0x2a, 0x0f, 0x02, 0xda, 0xb5, 0x53, 0x25, 0x82, 0xdb, 0xcc, 0xf7, 0xcd, - 0xce, 0xce, 0x7e, 0xdf, 0x0e, 0x34, 0x92, 0x79, 0x3f, 0x49, 0x85, 0x14, 0xd4, 0x48, 0xe6, 0xce, - 0x0f, 0x02, 0xc6, 0xe4, 0x86, 0x76, 0xa0, 0x1a, 0xe0, 0xda, 0x22, 0x36, 0xe9, 0xed, 0x33, 0x15, - 0xd2, 0xff, 0x61, 0x2f, 0xe7, 0xe1, 0x0a, 0x2d, 0x43, 0x63, 0x45, 0x42, 0x9f, 0x41, 0x73, 0x95, - 0x61, 0x3a, 0x8b, 0x50, 0x72, 0xab, 0xaa, 0x99, 0x86, 0x02, 0xae, 0x50, 0x72, 0x6a, 0x41, 0x3d, - 0xc7, 0x34, 0xf3, 0x45, 0x6c, 0xd5, 0x6c, 0xd2, 0xab, 0xb1, 0x4d, 0x4a, 0x5f, 0x00, 0xe0, 0x7d, - 0xe2, 0xa7, 0x98, 0xcd, 0xb8, 0xb4, 0xf6, 0x34, 0xd9, 0x2c, 0x91, 0x81, 0xa4, 0x14, 0x6a, 0xba, - 0xa1, 0xa9, 0x1b, 0xea, 0x58, 0xdd, 0x94, 0xc9, 0x14, 0x79, 0x34, 0xf3, 0x3d, 0x0b, 0x6c, 0xd2, - 0x3b, 0x60, 0x8d, 0x02, 0x18, 0x7b, 0xf4, 0x25, 0xb4, 0x4a, 0xd2, 0x13, 0x31, 0x5a, 0x2d, 0x9b, - 0xf4, 0x1a, 0x0c, 0x0a, 0xe8, 0x42, 0xc4, 0xe8, 0xd8, 0x60, 0x4e, 0x6e, 0x2e, 0xfd, 0x4c, 0xd2, - 0x23, 0x30, 0x82, 0xdc, 0x22, 0x76, 0xb5, 0xd7, 0x3a, 0x35, 0xfb, 0xc9, 0xbc, 0x3f, 0xb9, 0x61, - 0x46, 0x90, 0x3b, 0x03, 0xf8, 0xef, 0x8a, 0xc7, 0xfe, 0x02, 0x33, 0x39, 0x5c, 0xf2, 0xf8, 0x0e, - 0x3f, 0xa3, 0xa4, 0x27, 0x50, 0x77, 0x75, 0x92, 0x95, 0x27, 0xa8, 0x3a, 0xb1, 0x5b, 0xc7, 0x36, - 0x25, 0xce, 0x6f, 0x02, 0xed, 0x5d, 0x8e, 0xb6, 0xc1, 0x18, 0x7b, 0x5a, 0xc6, 0x1a, 0x33, 0xc6, - 0x1e, 0x3d, 0x01, 0x63, 0x9a, 0x68, 0x09, 0xdb, 0xa7, 0xcf, 0xff, 0xee, 0xd5, 0x9f, 0x26, 0x98, - 0x72, 0xe9, 0x8b, 0x98, 0x19, 0xd3, 0x44, 0x69, 0x7e, 0x89, 0x39, 0x86, 0x5a, 0xd9, 0x03, 0x56, - 0x24, 0xf4, 0x09, 0x98, 0x01, 0xae, 0x95, 0x0c, 0x85, 0xaa, 0x7b, 0x01, 0xae, 0xc7, 0x1e, 0xfd, - 0x00, 0x87, 0x18, 0xbb, 0xe9, 0x3a, 0x51, 0xc7, 0x67, 0x3c, 0xbc, 0x13, 0x5a, 0xd8, 0x76, 0x31, - 0xf3, 0xe8, 0x91, 0x1a, 0x84, 0x77, 0x82, 0xb5, 0x71, 0x27, 0xa7, 0x36, 0xb4, 0x5c, 0x11, 0x25, - 0x29, 0x66, 0xda, 0x2e, 0x53, 0xdf, 0xb7, 0x0d, 0x39, 0xaf, 0xa1, 0xf9, 0x38, 0x1c, 0x05, 0x30, - 0x87, 0x6c, 0x34, 0xb8, 0x1e, 0x75, 0x2a, 0x2a, 0xbe, 0x18, 0x5d, 0x8e, 0xae, 0x47, 0x1d, 0xe2, - 0x8c, 0xa1, 0x75, 0x1e, 0x0a, 0x37, 0x98, 0x2e, 0x16, 0x19, 0xca, 0x7f, 0xfc, 0xa2, 0x23, 0x30, - 0x85, 0xe6, 0xb4, 0x06, 0x07, 0xac, 0xcc, 0x54, 0x65, 0x88, 0x71, 0xf9, 0x4e, 0x15, 0x3a, 0x5f, - 0x00, 0xae, 0xf9, 0x3c, 0xc4, 0x71, 0xec, 0xe1, 0x3d, 0x7d, 0x0b, 0xf5, 0xa2, 0x72, 0x63, 0xc4, - 0xa1, 0x7a, 0xd4, 0xd6, 0x5d, 0x6c, 0xc3, 0xd3, 0x57, 0xb0, 0x3f, 0x0f, 0x85, 0x88, 0x66, 0x0b, - 0x3f, 0x94, 0x98, 0x96, 0xff, 0xb5, 0xa5, 0xb1, 0x8f, 0x1a, 0x72, 0x04, 0x34, 0x86, 0x4b, 0x74, - 0x83, 0x6c, 0x15, 0xd1, 0x63, 0xa8, 0x69, 0xad, 0x88, 0xd6, 0xea, 0x48, 0xb5, 0xdd, 0x70, 0x7d, - 0x25, 0x4d, 0xea, 0xcb, 0x65, 0xc4, 0x74, 0x8d, 0x9a, 0x32, 0x5b, 0x45, 0xba, 0x63, 0x8d, 0xa9, - 0xd0, 0x79, 0x03, 0xcd, 0xc7, 0xa2, 0x42, 0x95, 0xe1, 0xd9, 0xe9, 0xb0, 0x53, 0xa1, 0xfb, 0xd0, - 0xb8, 0xbd, 0xfd, 0xc4, 0xb3, 0xe5, 0xfb, 0x77, 0x1d, 0xe2, 0xb8, 0x50, 0xbf, 0xe0, 0x92, 0x4f, - 0x70, 0xbd, 0xe5, 0x1e, 0xd9, 0x76, 0x8f, 0x42, 0xcd, 0xe3, 0x92, 0x97, 0xd3, 0xea, 0x58, 0x7d, - 0x1e, 0x3f, 0x2f, 0xb7, 0xca, 0xf0, 0x73, 0xb5, 0x35, 0x6e, 0x8a, 0x5c, 0xa2, 0xa7, 0xb6, 0x46, - 0x99, 0x5f, 0x65, 0xcd, 0x12, 0x19, 0xc8, 0xe3, 0xa7, 0xd0, 0xde, 0x75, 0x99, 0xd6, 0xa1, 0xca, - 0x31, 0xeb, 0x54, 0xce, 0x3b, 0xdf, 0x1e, 0xba, 0xe4, 0xfb, 0x43, 0x97, 0xfc, 0x7c, 0xe8, 0x92, - 0xaf, 0xbf, 0xba, 0x95, 0xb9, 0xa9, 0x57, 0xfe, 0xec, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x5d, - 0x5f, 0x6a, 0x47, 0xfe, 0x03, 0x00, 0x00, + // 653 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x54, 0xcf, 0x6e, 0xda, 0x4e, + 0x10, 0x66, 0x8d, 0x63, 0x60, 0x08, 0x84, 0xdf, 0xea, 0xd7, 0xc8, 0x55, 0x5b, 0x4a, 0x5d, 0x45, + 0xa2, 0x51, 0xc4, 0x21, 0xa9, 0x7a, 0xe9, 0x89, 0x10, 0xaa, 0x22, 0x12, 0x21, 0x6d, 0xa2, 0x28, + 0x37, 0xb4, 0xe0, 0x21, 0x58, 0xf8, 0x9f, 0xbc, 0x8b, 0x15, 0x72, 0xeb, 0x5b, 0xf4, 0x91, 0x7a, + 0xec, 0xa1, 0x0f, 0x50, 0xa5, 0x0f, 0xd2, 0x6a, 0xd7, 0x06, 0x81, 0xda, 0xdb, 0xcc, 0xf7, 0xcd, + 0xee, 0xe7, 0xf9, 0x66, 0xd6, 0x50, 0x8e, 0x27, 0x9d, 0x38, 0x89, 0x64, 0x44, 0x8d, 0x78, 0xe2, + 0xfc, 0x20, 0x60, 0x0c, 0x6f, 0x69, 0x03, 0x8a, 0x0b, 0x5c, 0xd9, 0xa4, 0x45, 0xda, 0xfb, 0x4c, + 0x85, 0xf4, 0x7f, 0xd8, 0x4b, 0xb9, 0xbf, 0x44, 0xdb, 0xd0, 0x58, 0x96, 0xd0, 0x17, 0x50, 0x59, + 0x0a, 0x4c, 0xc6, 0x01, 0x4a, 0x6e, 0x17, 0x35, 0x53, 0x56, 0xc0, 0x15, 0x4a, 0x4e, 0x6d, 0x28, + 0xa5, 0x98, 0x08, 0x2f, 0x0a, 0x6d, 0xb3, 0x45, 0xda, 0x26, 0x5b, 0xa7, 0xf4, 0x15, 0x00, 0x3e, + 0xc4, 0x5e, 0x82, 0x62, 0xcc, 0xa5, 0xbd, 0xa7, 0xc9, 0x4a, 0x8e, 0x74, 0x25, 0xa5, 0x60, 0xea, + 0x0b, 0x2d, 0x7d, 0xa1, 0x8e, 0x95, 0x92, 0x90, 0x09, 0xf2, 0x60, 0xec, 0xb9, 0x36, 0xb4, 0x48, + 0xbb, 0xc6, 0xca, 0x19, 0x30, 0x70, 0xe9, 0x6b, 0xa8, 0xe6, 0xa4, 0x1b, 0x85, 0x68, 0x57, 0x5b, + 0xa4, 0x5d, 0x66, 0x90, 0x41, 0x17, 0x51, 0x88, 0x4e, 0x0b, 0xac, 0xe1, 0xed, 0xa5, 0x27, 0x24, + 0x3d, 0x04, 0x63, 0x91, 0xda, 0xa4, 0x55, 0x6c, 0x57, 0x4f, 0xad, 0x4e, 0x3c, 0xe9, 0x0c, 0x6f, + 0x99, 0xb1, 0x48, 0x9d, 0x2e, 0xfc, 0x77, 0xc5, 0x43, 0x6f, 0x86, 0x42, 0xf6, 0xe6, 0x3c, 0xbc, + 0xc7, 0x6b, 0x94, 0xf4, 0x04, 0x4a, 0x53, 0x9d, 0x88, 0xfc, 0x04, 0x55, 0x27, 0x76, 0xeb, 0xd8, + 0xba, 0xc4, 0xf9, 0x4d, 0xa0, 0xbe, 0xcb, 0xd1, 0x3a, 0x18, 0x03, 0x57, 0xdb, 0x68, 0x32, 0x63, + 0xe0, 0xd2, 0x13, 0x30, 0x46, 0xb1, 0xb6, 0xb0, 0x7e, 0xfa, 0xf2, 0xef, 0xbb, 0x3a, 0xa3, 0x18, + 0x13, 0x2e, 0xbd, 0x28, 0x64, 0xc6, 0x28, 0x56, 0x9e, 0x5f, 0x62, 0x8a, 0xbe, 0x76, 0xb6, 0xc6, + 0xb2, 0x84, 0x3e, 0x03, 0x6b, 0x81, 0x2b, 0x65, 0x43, 0xe6, 0xea, 0xde, 0x02, 0x57, 0x03, 0x97, + 0x7e, 0x84, 0x03, 0x0c, 0xa7, 0xc9, 0x2a, 0x56, 0xc7, 0xc7, 0xdc, 0xbf, 0x8f, 0xb4, 0xb1, 0xf5, + 0xec, 0x9b, 0xfb, 0x1b, 0xaa, 0xeb, 0xdf, 0x47, 0xac, 0x8e, 0x3b, 0x39, 0x6d, 0x41, 0x75, 0x1a, + 0x05, 0x71, 0x82, 0x42, 0x8f, 0xcb, 0xd2, 0x7a, 0xdb, 0x90, 0xf3, 0x16, 0x2a, 0x9b, 0x8f, 0xa3, + 0x00, 0x56, 0x8f, 0xf5, 0xbb, 0x37, 0xfd, 0x46, 0x41, 0xc5, 0x17, 0xfd, 0xcb, 0xfe, 0x4d, 0xbf, + 0x41, 0x9c, 0x01, 0x54, 0xcf, 0xfd, 0x68, 0xba, 0x18, 0xcd, 0x66, 0x02, 0xe5, 0x3f, 0xb6, 0xe8, + 0x10, 0xac, 0x48, 0x73, 0xda, 0x83, 0x1a, 0xcb, 0x33, 0x55, 0xe9, 0x63, 0x98, 0xf7, 0xa9, 0x42, + 0xe7, 0x0b, 0x01, 0xb8, 0xe1, 0x13, 0x1f, 0x07, 0xa1, 0x8b, 0x0f, 0xf4, 0x1d, 0x94, 0xb2, 0xd2, + 0xf5, 0x24, 0x0e, 0x54, 0x57, 0x5b, 0x62, 0x6c, 0xcd, 0xd3, 0x37, 0xb0, 0x3f, 0xf1, 0xa3, 0x28, + 0x18, 0xcf, 0x3c, 0x5f, 0x62, 0x92, 0x2f, 0x6c, 0x55, 0x63, 0x9f, 0x34, 0x44, 0x8f, 0xa0, 0x8e, + 0x42, 0x7a, 0x01, 0x97, 0xe8, 0x8e, 0x85, 0xf7, 0x88, 0x5a, 0xd9, 0x64, 0xb5, 0x0d, 0x7a, 0xed, + 0x3d, 0xa2, 0x13, 0x41, 0xb9, 0x37, 0xc7, 0xe9, 0x42, 0x2c, 0x03, 0x7a, 0x0c, 0xa6, 0xf6, 0x94, + 0x68, 0x4f, 0x0f, 0x95, 0xfa, 0x9a, 0xeb, 0x28, 0x0b, 0x13, 0x4f, 0xce, 0x03, 0xa6, 0x6b, 0x54, + 0x37, 0x62, 0x19, 0x68, 0x61, 0x93, 0xa9, 0xd0, 0x39, 0x82, 0xca, 0xa6, 0x28, 0x73, 0xaf, 0x77, + 0x76, 0xda, 0x6b, 0x14, 0xe8, 0x3e, 0x94, 0xef, 0xee, 0x3e, 0x73, 0x31, 0xff, 0xf0, 0xbe, 0x41, + 0x9c, 0x29, 0x94, 0x2e, 0xb8, 0xe4, 0x43, 0x5c, 0x6d, 0x4d, 0x99, 0x6c, 0x4f, 0x99, 0x82, 0xe9, + 0x72, 0xc9, 0xf3, 0xa6, 0x74, 0xac, 0x96, 0xcc, 0x4b, 0xf3, 0xd7, 0x67, 0x78, 0xa9, 0x7a, 0x5d, + 0xd3, 0x04, 0x75, 0x6f, 0x5c, 0xea, 0x25, 0x29, 0xb2, 0x4a, 0x8e, 0x74, 0xe5, 0xf1, 0x73, 0xa8, + 0xef, 0x6e, 0x03, 0x2d, 0x41, 0x91, 0xa3, 0x68, 0x14, 0xce, 0x1b, 0xdf, 0x9e, 0x9a, 0xe4, 0xfb, + 0x53, 0x93, 0xfc, 0x7c, 0x6a, 0x92, 0xaf, 0xbf, 0x9a, 0x85, 0x89, 0xa5, 0x7f, 0x0d, 0x67, 0x7f, + 0x02, 0x00, 0x00, 0xff, 0xff, 0x8f, 0xf6, 0xa8, 0xfe, 0x26, 0x04, 0x00, 0x00, } func (m *KV) Marshal() (dAtA []byte, err error) { @@ -972,6 +981,11 @@ func (m *TableIndex) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.EstimatedSize != 0 { + i = encodeVarintPb(dAtA, i, uint64(m.EstimatedSize)) + i-- + dAtA[i] = 0x18 + } if len(m.BloomFilter) > 0 { i -= len(m.BloomFilter) copy(dAtA[i:], m.BloomFilter) @@ -1239,6 +1253,9 @@ func (m *TableIndex) Size() (n int) { if l > 0 { n += 1 + l + sovPb(uint64(l)) } + if m.EstimatedSize != 0 { + n += 1 + sovPb(uint64(m.EstimatedSize)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -2129,6 +2146,25 @@ func (m *TableIndex) Unmarshal(dAtA []byte) error { m.BloomFilter = []byte{} } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EstimatedSize", wireType) + } + m.EstimatedSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EstimatedSize |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPb(dAtA[iNdEx:]) diff --git a/pb/pb.proto b/pb/pb.proto index 4d91eec04..384125973 100644 --- a/pb/pb.proto +++ b/pb/pb.proto @@ -43,7 +43,7 @@ message ManifestChangeSet { } enum EncryptionAlgo { - aes = 0; + aes = 0; } message ManifestChange { @@ -68,6 +68,7 @@ message BlockOffset { message TableIndex { repeated BlockOffset offsets = 1; bytes bloom_filter = 2; + uint64 estimated_size = 3; } message Checksum { @@ -84,4 +85,4 @@ message DataKey { bytes data = 2; bytes iv = 3; int64 created_at = 4; -} \ No newline at end of file +} diff --git a/stream_writer.go b/stream_writer.go index 38cc9730c..c63e904b3 100644 --- a/stream_writer.go +++ b/stream_writer.go @@ -362,7 +362,11 @@ func (w *sortedWriter) Add(key []byte, vs y.ValueStruct) error { } w.lastKey = y.SafeCopy(w.lastKey, key) - w.builder.Add(key, vs) + var vp valuePointer + if vs.Meta&bitValuePointer > 0 { + vp.Decode(vs.Value) + } + w.builder.Add(key, vs, vp.Len) return nil } diff --git a/table/builder.go b/table/builder.go index 064b9e4fa..26519494b 100644 --- a/table/builder.go +++ b/table/builder.go @@ -69,11 +69,9 @@ type Builder struct { baseKey []byte // Base key for the current block. baseOffset uint32 // Offset for the current block. entryOffsets []uint32 // Offsets of entries present in current block. - - tableIndex *pb.TableIndex - keyHashes []uint64 - - opt *Options + tableIndex *pb.TableIndex + keyHashes []uint64 // Used for building the bloomfilter. + opt *Options } // NewTableBuilder makes a new TableBuilder. @@ -103,7 +101,7 @@ func (b *Builder) keyDiff(newKey []byte) []byte { return newKey[i:] } -func (b *Builder) addHelper(key []byte, v y.ValueStruct) { +func (b *Builder) addHelper(key []byte, v y.ValueStruct, vpLen uint64) { b.keyHashes = append(b.keyHashes, farm.Fingerprint64(y.ParseKey(key))) // diffKey stores the difference of key with baseKey. @@ -131,6 +129,10 @@ func (b *Builder) addHelper(key []byte, v y.ValueStruct) { b.buf.Write(diffKey) // We only need to store the key difference. v.EncodeTo(b.buf) + // Size of KV on SST. + sstSz := uint64(uint32(headerSize) + uint32(len(diffKey)) + v.EncodedSize()) + // Total estimated size = size on SST + size on vlog (length of value pointer). + b.tableIndex.EstimatedSize += (sstSz + vpLen) } /* @@ -210,7 +212,7 @@ func (b *Builder) shouldFinishBlock(key []byte, value y.ValueStruct) bool { } // Add adds a key-value pair to the block. -func (b *Builder) Add(key []byte, value y.ValueStruct) { +func (b *Builder) Add(key []byte, value y.ValueStruct, valueLen uint32) { if b.shouldFinishBlock(key, value) { b.finishBlock() // Start a new block. Initialize the block. @@ -219,7 +221,7 @@ func (b *Builder) Add(key []byte, value y.ValueStruct) { b.baseOffset = uint32(b.buf.Len()) b.entryOffsets = b.entryOffsets[:0] } - b.addHelper(key, value) + b.addHelper(key, value, uint64(valueLen)) } // TODO: vvv this was the comment on ReachedCapacity. diff --git a/table/builder_test.go b/table/builder_test.go index 26886633c..3af2ce358 100644 --- a/table/builder_test.go +++ b/table/builder_test.go @@ -74,7 +74,7 @@ func TestTableIndex(t *testing.T) { blockCount++ blockFirstKeys = append(blockFirstKeys, k) } - builder.Add(k, vs) + builder.Add(k, vs, 0) } _, err = f.Write(builder.Finish()) require.NoError(t, err, "unable to write to file") @@ -129,7 +129,7 @@ func BenchmarkBuilder(b *testing.B) { builder := NewTableBuilder(opts) for i := 0; i < keysCount; i++ { - builder.Add(key(i), vs) + builder.Add(key(i), vs, 0) } _ = builder.Finish() diff --git a/table/table.go b/table/table.go index 9a9140119..d68169384 100644 --- a/table/table.go +++ b/table/table.go @@ -99,6 +99,8 @@ type Table struct { bf *z.Bloom Checksum []byte + // Stores the total size of key-values stored in this table (including the size on vlog). + estimatedSize uint64 IsInmemory bool // Set to true if the table is on level 0 and opened in memory. opt *Options @@ -351,6 +353,7 @@ func (t *Table) readIndex() error { err := proto.Unmarshal(data, &index) y.Check(err) + t.estimatedSize = index.EstimatedSize t.bf = z.JSONUnmarshal(index.BloomFilter) t.blockIndex = index.Offsets return nil @@ -439,6 +442,10 @@ func (t *Table) blockCacheKey(idx int) uint64 { return (t.ID() << 32) | uint64(idx) } +// EstimatedSize returns the total size of key-values stored in this table (including the +// disk space occupied on the value log). +func (t *Table) EstimatedSize() uint64 { return t.estimatedSize } + // Size is its file size in bytes func (t *Table) Size() int64 { return int64(t.tableSize) } diff --git a/table/table_test.go b/table/table_test.go index 597595d48..82bddf591 100644 --- a/table/table_test.go +++ b/table/table_test.go @@ -90,7 +90,7 @@ func buildTable(t *testing.T, keyValues [][]string, opts Options) *os.File { }) for _, kv := range keyValues { y.AssertTrue(len(kv) == 2) - b.Add(y.KeyWithTs([]byte(kv[0]), 0), y.ValueStruct{Value: []byte(kv[1]), Meta: 'A', UserMeta: 0}) + b.Add(y.KeyWithTs([]byte(kv[0]), 0), y.ValueStruct{Value: []byte(kv[1]), Meta: 'A', UserMeta: 0}, 0) } _, err = f.Write(b.Finish()) require.NoError(t, err, "writing to file failed") @@ -710,7 +710,7 @@ func TestTableBigValues(t *testing.T) { for i := 0; i < n; i++ { key := y.KeyWithTs([]byte(key("", i)), 0) vs := y.ValueStruct{Value: value(i)} - builder.Add(key, vs) + builder.Add(key, vs, 0) } _, err = f.Write(builder.Finish()) @@ -793,7 +793,7 @@ func BenchmarkReadAndBuild(b *testing.B) { defer it.Close() for it.seekToFirst(); it.Valid(); it.next() { vs := it.Value() - newBuilder.Add(it.Key(), vs) + newBuilder.Add(it.Key(), vs, 0) } newBuilder.Finish() }() @@ -822,7 +822,7 @@ func BenchmarkReadMerged(b *testing.B) { // id := i*tableSize+j (not interleaved) k := fmt.Sprintf("%016x", id) v := fmt.Sprintf("%d", id) - builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0}) + builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0}, 0) } _, err = f.Write(builder.Finish()) require.NoError(b, err, "unable to write to file") @@ -913,7 +913,7 @@ func getTableForBenchmarks(b *testing.B, count int, cache *ristretto.Cache) *Tab for i := 0; i < count; i++ { k := fmt.Sprintf("%016x", i) v := fmt.Sprintf("%d", i) - builder.Add([]byte(k), y.ValueStruct{Value: []byte(v)}) + builder.Add([]byte(k), y.ValueStruct{Value: []byte(v)}, 0) } _, err = f.Write(builder.Finish()) @@ -927,3 +927,13 @@ func TestMain(m *testing.M) { rand.Seed(time.Now().UTC().UnixNano()) os.Exit(m.Run()) } + +func TestOpenKVSize(t *testing.T) { + opts := getTestTableOptions() + table, err := OpenTable(buildTestTable(t, "foo", 1, opts), opts) + require.NoError(t, err) + + // The following values might change if the table/header structure is changed. + var entrySize uint64 = 15 /* DiffKey len */ + 4 /* Header Size */ + 4 /* Encoded vp */ + require.Equal(t, entrySize, table.EstimatedSize()) +}