From f4ef8ad98096e954e93e245e424466c482f91783 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Fri, 5 Aug 2022 15:13:07 -0700 Subject: [PATCH 1/3] Rename CheckpointIndex -> BitmapIndex to reflect its usage --- exp/lighthorizon/index/backend/gzip_test.go | 4 +- exp/lighthorizon/index/backend/s3.go | 2 +- .../index/cmd/batch/reduce/main.go | 2 +- exp/lighthorizon/index/store.go | 8 +- exp/lighthorizon/index/types/bitmap.go | 146 +++++++++--------- exp/lighthorizon/index/types/bitmap_test.go | 86 +++++------ gxdr/xdr_generated.go | 26 ++-- xdr/Stellar-lighthorizon.x | 6 +- xdr/xdr_generated.go | 42 ++--- 9 files changed, 161 insertions(+), 161 deletions(-) diff --git a/exp/lighthorizon/index/backend/gzip_test.go b/exp/lighthorizon/index/backend/gzip_test.go index f7abb168d2..730e13185d 100644 --- a/exp/lighthorizon/index/backend/gzip_test.go +++ b/exp/lighthorizon/index/backend/gzip_test.go @@ -13,8 +13,8 @@ import ( ) func TestGzipRoundtrip(t *testing.T) { - index := &types.CheckpointIndex{} - anotherIndex := &types.CheckpointIndex{} + index := &types.BitmapIndex{} + anotherIndex := &types.BitmapIndex{} for i := 0; i < 100+rand.Intn(1000); i++ { index.SetActive(uint32(rand.Intn(10_000))) anotherIndex.SetActive(uint32(rand.Intn(10_000))) diff --git a/exp/lighthorizon/index/backend/s3.go b/exp/lighthorizon/index/backend/s3.go index 996b8f3cec..a4f5a7e751 100644 --- a/exp/lighthorizon/index/backend/s3.go +++ b/exp/lighthorizon/index/backend/s3.go @@ -173,7 +173,7 @@ func (s *S3Backend) Read(account string) (types.NamedIndices, error) { if n == 0 { return nil, os.ErrNotExist } - var indexes map[string]*types.CheckpointIndex + var indexes map[string]*types.BitmapIndex indexes, _, err = readGzippedFrom(bytes.NewReader(b.Bytes())) if err != nil { log.Errorf("Unable to parse %s: %v", account, err) diff --git a/exp/lighthorizon/index/cmd/batch/reduce/main.go b/exp/lighthorizon/index/cmd/batch/reduce/main.go index 5b1bfa8dae..a6e82861da 100644 --- a/exp/lighthorizon/index/cmd/batch/reduce/main.go +++ b/exp/lighthorizon/index/cmd/batch/reduce/main.go @@ -354,7 +354,7 @@ func (cfg *ReduceConfig) shouldProcessTx(txPrefix byte, routineIndex uint32) boo // For every index that exists in `dest`, finds the corresponding index in // `source` and merges it into `dest`'s version. -func mergeIndices(dest, source map[string]*types.CheckpointIndex) error { +func mergeIndices(dest, source map[string]*types.BitmapIndex) error { for name, index := range dest { // The source doesn't contain this particular index. // diff --git a/exp/lighthorizon/index/store.go b/exp/lighthorizon/index/store.go index ddea9e70f4..c10f9f5018 100644 --- a/exp/lighthorizon/index/store.go +++ b/exp/lighthorizon/index/store.go @@ -227,12 +227,12 @@ func (s *store) AddParticipantsToIndexesNoBackend(checkpoint uint32, index strin var err error for _, participant := range participants { if _, ok := s.indexes[participant]; !ok { - s.indexes[participant] = map[string]*types.CheckpointIndex{} + s.indexes[participant] = map[string]*types.BitmapIndex{} } ind, ok := s.indexes[participant][index] if !ok { - ind = &types.CheckpointIndex{} + ind = &types.BitmapIndex{} s.indexes[participant][index] = ind } @@ -269,7 +269,7 @@ func (s *store) AddParticipantsToIndexes(checkpoint uint32, index string, partic return nil } -func (s *store) getCreateIndex(account, id string) (*types.CheckpointIndex, error) { +func (s *store) getCreateIndex(account, id string) (*types.BitmapIndex, error) { s.mutex.Lock() defer s.mutex.Unlock() defer s.approximateWorkingSet() @@ -295,7 +295,7 @@ func (s *store) getCreateIndex(account, id string) (*types.CheckpointIndex, erro ind, ok = accountIndexes[id] if !ok { // Not found anywhere, make a new one. - ind = &types.CheckpointIndex{} + ind = &types.BitmapIndex{} accountIndexes[id] = ind } diff --git a/exp/lighthorizon/index/types/bitmap.go b/exp/lighthorizon/index/types/bitmap.go index 346ede80a7..31046c82f2 100644 --- a/exp/lighthorizon/index/types/bitmap.go +++ b/exp/lighthorizon/index/types/bitmap.go @@ -10,17 +10,17 @@ import ( const CheckpointIndexVersion = 1 -type CheckpointIndex struct { - mutex sync.RWMutex - bitmap []byte - firstCheckpoint uint32 - lastCheckpoint uint32 +type BitmapIndex struct { + mutex sync.RWMutex + bitmap []byte + firstBit uint32 + lastBit uint32 } -type NamedIndices map[string]*CheckpointIndex +type NamedIndices map[string]*BitmapIndex -func NewCheckpointIndex(b []byte) (*CheckpointIndex, error) { - xdrCheckpoint := xdr.CheckpointIndex{} +func NewCheckpointIndex(b []byte) (*BitmapIndex, error) { + xdrCheckpoint := xdr.BitmapIndex{} err := xdrCheckpoint.UnmarshalBinary(b) if err != nil { return nil, err @@ -29,83 +29,83 @@ func NewCheckpointIndex(b []byte) (*CheckpointIndex, error) { return NewCheckpointIndexFromXDR(xdrCheckpoint), nil } -func NewCheckpointIndexFromXDR(index xdr.CheckpointIndex) *CheckpointIndex { - return &CheckpointIndex{ - bitmap: index.Bitmap[:], - firstCheckpoint: uint32(index.FirstCheckpoint), - lastCheckpoint: uint32(index.LastCheckpoint), +func NewCheckpointIndexFromXDR(index xdr.BitmapIndex) *BitmapIndex { + return &BitmapIndex{ + bitmap: index.Bitmap[:], + firstBit: uint32(index.FirstBit), + lastBit: uint32(index.LastBit), } } -func (i *CheckpointIndex) Size() int { +func (i *BitmapIndex) Size() int { return len(i.bitmap) } -func (i *CheckpointIndex) SetActive(checkpoint uint32) error { +func (i *BitmapIndex) SetActive(index uint32) error { i.mutex.Lock() defer i.mutex.Unlock() - return i.setActive(checkpoint) + return i.setActive(index) } -func bitShiftLeft(checkpoint uint32) byte { - if checkpoint%8 == 0 { +func bitShiftLeft(index uint32) byte { + if index%8 == 0 { return 1 } else { - return byte(1) << (8 - checkpoint%8) + return byte(1) << (8 - index%8) } } -func (i *CheckpointIndex) rangeFirstCheckpoint() uint32 { - return (i.firstCheckpoint-1)/8*8 + 1 +func (i *BitmapIndex) rangeFirstBit() uint32 { + return (i.firstBit-1)/8*8 + 1 } -func (i *CheckpointIndex) rangeLastCheckpoint() uint32 { - return i.rangeFirstCheckpoint() + uint32(len(i.bitmap))*8 - 1 +func (i *BitmapIndex) rangeLastBit() uint32 { + return i.rangeFirstBit() + uint32(len(i.bitmap))*8 - 1 } -func (i *CheckpointIndex) setActive(checkpoint uint32) error { - if i.firstCheckpoint == 0 { - i.firstCheckpoint = checkpoint - i.lastCheckpoint = checkpoint - b := bitShiftLeft(checkpoint) +func (i *BitmapIndex) setActive(index uint32) error { + if i.firstBit == 0 { + i.firstBit = index + i.lastBit = index + b := bitShiftLeft(index) i.bitmap = []byte{b} } else { - if checkpoint >= i.rangeFirstCheckpoint() && checkpoint <= i.rangeLastCheckpoint() { + if index >= i.rangeFirstBit() && index <= i.rangeLastBit() { // Update the bit in existing range - b := bitShiftLeft(checkpoint) - loc := (checkpoint - i.rangeFirstCheckpoint()) / 8 + b := bitShiftLeft(index) + loc := (index - i.rangeFirstBit()) / 8 i.bitmap[loc] = i.bitmap[loc] | b - if checkpoint < i.firstCheckpoint { - i.firstCheckpoint = checkpoint + if index < i.firstBit { + i.firstBit = index } - if checkpoint > i.lastCheckpoint { - i.lastCheckpoint = checkpoint + if index > i.lastBit { + i.lastBit = index } } else { // Expand the bitmap - if checkpoint < i.rangeFirstCheckpoint() { + if index < i.rangeFirstBit() { // ...to the left - c := (i.rangeFirstCheckpoint() - checkpoint) / 8 - if (i.rangeFirstCheckpoint()-checkpoint)%8 != 0 { + c := (i.rangeFirstBit() - index) / 8 + if (i.rangeFirstBit()-index)%8 != 0 { c++ } newBytes := make([]byte, c) i.bitmap = append(newBytes, i.bitmap...) - b := bitShiftLeft(checkpoint) + b := bitShiftLeft(index) i.bitmap[0] = i.bitmap[0] | b - i.firstCheckpoint = checkpoint - } else if checkpoint > i.rangeLastCheckpoint() { + i.firstBit = index + } else if index > i.rangeLastBit() { // ... to the right - newBytes := make([]byte, (checkpoint-i.rangeLastCheckpoint())/8+1) + newBytes := make([]byte, (index-i.rangeLastBit())/8+1) i.bitmap = append(i.bitmap, newBytes...) - b := bitShiftLeft(checkpoint) - loc := (checkpoint - i.rangeFirstCheckpoint()) / 8 + b := bitShiftLeft(index) + loc := (index - i.rangeFirstBit()) / 8 i.bitmap[loc] = i.bitmap[loc] | b - i.lastCheckpoint = checkpoint + i.lastBit = index } } } @@ -114,26 +114,26 @@ func (i *CheckpointIndex) setActive(checkpoint uint32) error { } //lint:ignore U1000 Ignore unused function temporarily -func (i *CheckpointIndex) isActive(checkpoint uint32) bool { - if checkpoint >= i.firstCheckpoint && checkpoint <= i.lastCheckpoint { - b := bitShiftLeft(checkpoint) - loc := (checkpoint - i.rangeFirstCheckpoint()) / 8 +func (i *BitmapIndex) isActive(index uint32) bool { + if index >= i.firstBit && index <= i.lastBit { + b := bitShiftLeft(index) + loc := (index - i.rangeFirstBit()) / 8 return i.bitmap[loc]&b != 0 } else { return false } } -func (i *CheckpointIndex) iterate(f func(checkpoint uint32)) error { +func (i *BitmapIndex) iterate(f func(index uint32)) error { i.mutex.RLock() defer i.mutex.RUnlock() - if i.firstCheckpoint == 0 { + if i.firstBit == 0 { return nil } - f(i.firstCheckpoint) - curr := i.firstCheckpoint + f(i.firstBit) + curr := i.firstBit for { var err error @@ -151,16 +151,16 @@ func (i *CheckpointIndex) iterate(f func(checkpoint uint32)) error { return nil } -func (i *CheckpointIndex) Merge(other *CheckpointIndex) error { +func (i *BitmapIndex) Merge(other *BitmapIndex) error { i.mutex.Lock() defer i.mutex.Unlock() var err error - other.iterate(func(checkpoint uint32) { + other.iterate(func(index uint32) { if err != nil { return } - err = i.setActive(checkpoint) + err = i.setActive(index) }) return err @@ -169,29 +169,29 @@ func (i *CheckpointIndex) Merge(other *CheckpointIndex) error { // NextActive returns the next checkpoint (inclusive) where this index is // active. "Inclusive" means that if the index is active at `checkpoint`, this // returns `checkpoint`. -func (i *CheckpointIndex) NextActive(checkpoint uint32) (uint32, error) { +func (i *BitmapIndex) NextActive(index uint32) (uint32, error) { i.mutex.RLock() defer i.mutex.RUnlock() - return i.nextActive(checkpoint) + return i.nextActive(index) } -func (i *CheckpointIndex) nextActive(checkpoint uint32) (uint32, error) { - if i.firstCheckpoint == 0 || checkpoint > i.lastCheckpoint { +func (i *BitmapIndex) nextActive(index uint32) (uint32, error) { + if i.firstBit == 0 || index > i.lastBit { // We're past the end. // TODO: Should this be an error? or how should we signal NONE here? return 0, io.EOF } - if checkpoint < i.firstCheckpoint { - checkpoint = i.firstCheckpoint + if index < i.firstBit { + index = i.firstBit } // Must be within the range, find the first non-zero after our start - loc := (checkpoint - i.rangeFirstCheckpoint()) / 8 + loc := (index - i.rangeFirstBit()) / 8 // Is it in the same byte? - if shift, ok := maxBitAfter(i.bitmap[loc], (checkpoint-1)%8); ok { - return i.rangeFirstCheckpoint() + (loc * 8) + shift, nil + if shift, ok := maxBitAfter(i.bitmap[loc], (index-1)%8); ok { + return i.rangeFirstBit() + (loc * 8) + shift, nil } // Scan bytes after @@ -199,7 +199,7 @@ func (i *CheckpointIndex) nextActive(checkpoint uint32) (uint32, error) { for ; loc < uint32(len(i.bitmap)); loc++ { // Find the offset of the set bit if shift, ok := maxBitAfter(i.bitmap[loc], 0); ok { - return i.rangeFirstCheckpoint() + (loc * 8) + shift, nil + return i.rangeFirstBit() + (loc * 8) + shift, nil } } @@ -223,18 +223,18 @@ func maxBitAfter(b byte, after uint32) (uint32, bool) { return 0, false } -func (i *CheckpointIndex) ToXDR() xdr.CheckpointIndex { +func (i *BitmapIndex) ToXDR() xdr.BitmapIndex { i.mutex.RLock() defer i.mutex.RUnlock() - return xdr.CheckpointIndex{ - FirstCheckpoint: xdr.Uint32(i.firstCheckpoint), - LastCheckpoint: xdr.Uint32(i.lastCheckpoint), - Bitmap: i.bitmap, + return xdr.BitmapIndex{ + FirstBit: xdr.Uint32(i.firstBit), + LastBit: xdr.Uint32(i.lastBit), + Bitmap: i.bitmap, } } -func (i *CheckpointIndex) Buffer() *bytes.Buffer { +func (i *BitmapIndex) Buffer() *bytes.Buffer { i.mutex.RLock() defer i.mutex.RUnlock() @@ -247,6 +247,6 @@ func (i *CheckpointIndex) Buffer() *bytes.Buffer { } // Flush flushes the index data to byte slice in index format. -func (i *CheckpointIndex) Flush() []byte { +func (i *BitmapIndex) Flush() []byte { return i.Buffer().Bytes() } diff --git a/exp/lighthorizon/index/types/bitmap_test.go b/exp/lighthorizon/index/types/bitmap_test.go index f131e38b83..d47ec58237 100644 --- a/exp/lighthorizon/index/types/bitmap_test.go +++ b/exp/lighthorizon/index/types/bitmap_test.go @@ -12,13 +12,13 @@ import ( func TestNewFromBytes(t *testing.T) { for i := uint32(1); i < 200; i++ { t.Run(fmt.Sprintf("New%d", i), func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(i) b := index.Flush() newIndex, err := NewCheckpointIndex(b) require.NoError(t, err) - assert.Equal(t, index.firstCheckpoint, newIndex.firstCheckpoint) - assert.Equal(t, index.lastCheckpoint, newIndex.lastCheckpoint) + assert.Equal(t, index.firstBit, newIndex.firstBit) + assert.Equal(t, index.lastBit, newIndex.lastBit) assert.Equal(t, index.bitmap, newIndex.bitmap) }) } @@ -51,85 +51,85 @@ func TestSetActive(t *testing.T) { for _, tt := range cases { t.Run(fmt.Sprintf("init_%d", tt.checkpoint), func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(tt.checkpoint) assert.Equal(t, tt.bitmap, index.bitmap) - assert.Equal(t, tt.rangeFirstCheckpoint, index.rangeFirstCheckpoint()) - assert.Equal(t, tt.checkpoint, index.firstCheckpoint) - assert.Equal(t, tt.checkpoint, index.lastCheckpoint) + assert.Equal(t, tt.rangeFirstCheckpoint, index.rangeFirstBit()) + assert.Equal(t, tt.checkpoint, index.firstBit) + assert.Equal(t, tt.checkpoint, index.lastBit) }) } // Update current bitmap right - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(1) - assert.Equal(t, uint32(1), index.firstCheckpoint) - assert.Equal(t, uint32(1), index.lastCheckpoint) + assert.Equal(t, uint32(1), index.firstBit) + assert.Equal(t, uint32(1), index.lastBit) index.SetActive(8) assert.Equal(t, []byte{0b1000_0001}, index.bitmap) - assert.Equal(t, uint32(1), index.firstCheckpoint) - assert.Equal(t, uint32(8), index.lastCheckpoint) + assert.Equal(t, uint32(1), index.firstBit) + assert.Equal(t, uint32(8), index.lastBit) // Update current bitmap left - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(8) - assert.Equal(t, uint32(8), index.firstCheckpoint) - assert.Equal(t, uint32(8), index.lastCheckpoint) + assert.Equal(t, uint32(8), index.firstBit) + assert.Equal(t, uint32(8), index.lastBit) index.SetActive(1) assert.Equal(t, []byte{0b1000_0001}, index.bitmap) - assert.Equal(t, uint32(1), index.firstCheckpoint) - assert.Equal(t, uint32(8), index.lastCheckpoint) + assert.Equal(t, uint32(1), index.firstBit) + assert.Equal(t, uint32(8), index.lastBit) - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(10) index.SetActive(9) index.SetActive(16) assert.Equal(t, []byte{0b1100_0001}, index.bitmap) - assert.Equal(t, uint32(9), index.firstCheckpoint) - assert.Equal(t, uint32(16), index.lastCheckpoint) + assert.Equal(t, uint32(9), index.firstBit) + assert.Equal(t, uint32(16), index.lastBit) // Expand bitmap to the left - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(10) index.SetActive(1) assert.Equal(t, []byte{0b1000_0000, 0b0100_0000}, index.bitmap) - assert.Equal(t, uint32(1), index.firstCheckpoint) - assert.Equal(t, uint32(10), index.lastCheckpoint) + assert.Equal(t, uint32(1), index.firstBit) + assert.Equal(t, uint32(10), index.lastBit) - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(17) index.SetActive(2) assert.Equal(t, []byte{0b0100_0000, 0b0000_0000, 0b1000_0000}, index.bitmap) - assert.Equal(t, uint32(2), index.firstCheckpoint) - assert.Equal(t, uint32(17), index.lastCheckpoint) + assert.Equal(t, uint32(2), index.firstBit) + assert.Equal(t, uint32(17), index.lastBit) // Expand bitmap to the right - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(1) index.SetActive(10) assert.Equal(t, []byte{0b1000_0000, 0b0100_0000}, index.bitmap) - assert.Equal(t, uint32(1), index.firstCheckpoint) - assert.Equal(t, uint32(10), index.lastCheckpoint) + assert.Equal(t, uint32(1), index.firstBit) + assert.Equal(t, uint32(10), index.lastBit) - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(2) index.SetActive(17) assert.Equal(t, []byte{0b0100_0000, 0b0000_0000, 0b1000_0000}, index.bitmap) - assert.Equal(t, uint32(2), index.firstCheckpoint) - assert.Equal(t, uint32(17), index.lastCheckpoint) + assert.Equal(t, uint32(2), index.firstBit) + assert.Equal(t, uint32(17), index.lastBit) - index = &CheckpointIndex{} + index = &BitmapIndex{} index.SetActive(17) index.SetActive(26) assert.Equal(t, []byte{0b1000_0000, 0b0100_0000}, index.bitmap) - assert.Equal(t, uint32(17), index.firstCheckpoint) - assert.Equal(t, uint32(26), index.lastCheckpoint) + assert.Equal(t, uint32(17), index.firstBit) + assert.Equal(t, uint32(26), index.lastBit) } func TestNextActive(t *testing.T) { t.Run("empty", func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} i, err := index.NextActive(0) assert.Equal(t, uint32(0), i) @@ -138,7 +138,7 @@ func TestNextActive(t *testing.T) { t.Run("one byte", func(t *testing.T) { t.Run("after last", func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(3) // 16 is well-past the end @@ -148,7 +148,7 @@ func TestNextActive(t *testing.T) { }) t.Run("only one bit in the byte", func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(1) i, err := index.NextActive(1) @@ -157,7 +157,7 @@ func TestNextActive(t *testing.T) { }) t.Run("only one bit in the byte (offset)", func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(9) i, err := index.NextActive(1) @@ -165,7 +165,7 @@ func TestNextActive(t *testing.T) { assert.Equal(t, uint32(9), i) }) - severalSet := &CheckpointIndex{} + severalSet := &BitmapIndex{} severalSet.SetActive(9) severalSet.SetActive(11) @@ -189,7 +189,7 @@ func TestNextActive(t *testing.T) { }) t.Run("many bytes", func(t *testing.T) { - index := &CheckpointIndex{} + index := &BitmapIndex{} index.SetActive(9) index.SetActive(129) @@ -248,11 +248,11 @@ func TestMaxBitAfter(t *testing.T) { } func TestMerge(t *testing.T) { - a := &CheckpointIndex{} + a := &BitmapIndex{} require.NoError(t, a.SetActive(9)) require.NoError(t, a.SetActive(129)) - b := &CheckpointIndex{} + b := &BitmapIndex{} require.NoError(t, b.SetActive(900)) require.NoError(t, b.SetActive(1000)) diff --git a/gxdr/xdr_generated.go b/gxdr/xdr_generated.go index 48c0f0a688..b28f1aeeeb 100644 --- a/gxdr/xdr_generated.go +++ b/gxdr/xdr_generated.go @@ -3088,10 +3088,10 @@ type HmacSha256Mac struct { Mac [32]byte } -type CheckpointIndex struct { - FirstCheckpoint Uint32 - LastCheckpoint Uint32 - Bitmap Value +type BitmapIndex struct { + FirstBit Uint32 + LastBit Uint32 + Bitmap Value } type TrieIndex struct { @@ -19522,21 +19522,21 @@ func (v *HmacSha256Mac) XdrRecurse(x XDR, name string) { } func XDR_HmacSha256Mac(v *HmacSha256Mac) *HmacSha256Mac { return v } -type XdrType_CheckpointIndex = *CheckpointIndex +type XdrType_BitmapIndex = *BitmapIndex -func (v *CheckpointIndex) XdrPointer() interface{} { return v } -func (CheckpointIndex) XdrTypeName() string { return "CheckpointIndex" } -func (v CheckpointIndex) XdrValue() interface{} { return v } -func (v *CheckpointIndex) XdrMarshal(x XDR, name string) { x.Marshal(name, v) } -func (v *CheckpointIndex) XdrRecurse(x XDR, name string) { +func (v *BitmapIndex) XdrPointer() interface{} { return v } +func (BitmapIndex) XdrTypeName() string { return "BitmapIndex" } +func (v BitmapIndex) XdrValue() interface{} { return v } +func (v *BitmapIndex) XdrMarshal(x XDR, name string) { x.Marshal(name, v) } +func (v *BitmapIndex) XdrRecurse(x XDR, name string) { if name != "" { name = x.Sprintf("%s.", name) } - x.Marshal(x.Sprintf("%sfirstCheckpoint", name), XDR_Uint32(&v.FirstCheckpoint)) - x.Marshal(x.Sprintf("%slastCheckpoint", name), XDR_Uint32(&v.LastCheckpoint)) + x.Marshal(x.Sprintf("%sfirstBit", name), XDR_Uint32(&v.FirstBit)) + x.Marshal(x.Sprintf("%slastBit", name), XDR_Uint32(&v.LastBit)) x.Marshal(x.Sprintf("%sbitmap", name), XDR_Value(&v.Bitmap)) } -func XDR_CheckpointIndex(v *CheckpointIndex) *CheckpointIndex { return v } +func XDR_BitmapIndex(v *BitmapIndex) *BitmapIndex { return v } type XdrType_TrieIndex = *TrieIndex diff --git a/xdr/Stellar-lighthorizon.x b/xdr/Stellar-lighthorizon.x index 0ef1d0cc9a..8955871cd1 100644 --- a/xdr/Stellar-lighthorizon.x +++ b/xdr/Stellar-lighthorizon.x @@ -8,9 +8,9 @@ namespace stellar { -struct CheckpointIndex { - uint32 firstCheckpoint; - uint32 lastCheckpoint; +struct BitmapIndex { + uint32 firstBit; + uint32 lastBit; Value bitmap; }; diff --git a/xdr/xdr_generated.go b/xdr/xdr_generated.go index 069436d74a..41af2e4436 100644 --- a/xdr/xdr_generated.go +++ b/xdr/xdr_generated.go @@ -35865,27 +35865,27 @@ func (s HmacSha256Mac) xdrType() {} var _ xdrType = (*HmacSha256Mac)(nil) -// CheckpointIndex is an XDR Struct defines as: +// BitmapIndex is an XDR Struct defines as: // -// struct CheckpointIndex { -// uint32 firstCheckpoint; -// uint32 lastCheckpoint; +// struct BitmapIndex { +// uint32 firstBit; +// uint32 lastBit; // Value bitmap; // }; // -type CheckpointIndex struct { - FirstCheckpoint Uint32 - LastCheckpoint Uint32 - Bitmap Value +type BitmapIndex struct { + FirstBit Uint32 + LastBit Uint32 + Bitmap Value } // EncodeTo encodes this value using the Encoder. -func (s *CheckpointIndex) EncodeTo(e *xdr.Encoder) error { +func (s *BitmapIndex) EncodeTo(e *xdr.Encoder) error { var err error - if err = s.FirstCheckpoint.EncodeTo(e); err != nil { + if err = s.FirstBit.EncodeTo(e); err != nil { return err } - if err = s.LastCheckpoint.EncodeTo(e); err != nil { + if err = s.LastBit.EncodeTo(e); err != nil { return err } if err = s.Bitmap.EncodeTo(e); err != nil { @@ -35894,18 +35894,18 @@ func (s *CheckpointIndex) EncodeTo(e *xdr.Encoder) error { return nil } -var _ decoderFrom = (*CheckpointIndex)(nil) +var _ decoderFrom = (*BitmapIndex)(nil) // DecodeFrom decodes this value using the Decoder. -func (s *CheckpointIndex) DecodeFrom(d *xdr.Decoder) (int, error) { +func (s *BitmapIndex) DecodeFrom(d *xdr.Decoder) (int, error) { var err error var n, nTmp int - nTmp, err = s.FirstCheckpoint.DecodeFrom(d) + nTmp, err = s.FirstBit.DecodeFrom(d) n += nTmp if err != nil { return n, fmt.Errorf("decoding Uint32: %s", err) } - nTmp, err = s.LastCheckpoint.DecodeFrom(d) + nTmp, err = s.LastBit.DecodeFrom(d) n += nTmp if err != nil { return n, fmt.Errorf("decoding Uint32: %s", err) @@ -35919,7 +35919,7 @@ func (s *CheckpointIndex) DecodeFrom(d *xdr.Decoder) (int, error) { } // MarshalBinary implements encoding.BinaryMarshaler. -func (s CheckpointIndex) MarshalBinary() ([]byte, error) { +func (s BitmapIndex) MarshalBinary() ([]byte, error) { b := bytes.Buffer{} e := xdr.NewEncoder(&b) err := s.EncodeTo(e) @@ -35927,7 +35927,7 @@ func (s CheckpointIndex) MarshalBinary() ([]byte, error) { } // UnmarshalBinary implements encoding.BinaryUnmarshaler. -func (s *CheckpointIndex) UnmarshalBinary(inp []byte) error { +func (s *BitmapIndex) UnmarshalBinary(inp []byte) error { r := bytes.NewReader(inp) d := xdr.NewDecoder(r) _, err := s.DecodeFrom(d) @@ -35935,15 +35935,15 @@ func (s *CheckpointIndex) UnmarshalBinary(inp []byte) error { } var ( - _ encoding.BinaryMarshaler = (*CheckpointIndex)(nil) - _ encoding.BinaryUnmarshaler = (*CheckpointIndex)(nil) + _ encoding.BinaryMarshaler = (*BitmapIndex)(nil) + _ encoding.BinaryUnmarshaler = (*BitmapIndex)(nil) ) // xdrType signals that this type is an type representing // representing XDR values defined by this package. -func (s CheckpointIndex) xdrType() {} +func (s BitmapIndex) xdrType() {} -var _ xdrType = (*CheckpointIndex)(nil) +var _ xdrType = (*BitmapIndex)(nil) // TrieIndex is an XDR Struct defines as: // From 9eb3cc72a64aa76661b526427832bf87b92ae9be Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Fri, 5 Aug 2022 15:15:47 -0700 Subject: [PATCH 2/3] Rename NextActive -> NextActiveBit to be descriptive --- exp/lighthorizon/index/store.go | 2 +- exp/lighthorizon/index/types/bitmap.go | 10 ++++----- exp/lighthorizon/index/types/bitmap_test.go | 24 ++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/exp/lighthorizon/index/store.go b/exp/lighthorizon/index/store.go index c10f9f5018..61355bd8cb 100644 --- a/exp/lighthorizon/index/store.go +++ b/exp/lighthorizon/index/store.go @@ -323,7 +323,7 @@ func (s *store) NextActive(account, indexId string, afterCheckpoint uint32) (uin if err != nil { return 0, err } - return ind.NextActive(afterCheckpoint) + return ind.NextActiveBit(afterCheckpoint) } func (s *store) getCreateTrieIndex(prefix string) (*types.TrieIndex, error) { diff --git a/exp/lighthorizon/index/types/bitmap.go b/exp/lighthorizon/index/types/bitmap.go index 31046c82f2..d250f92322 100644 --- a/exp/lighthorizon/index/types/bitmap.go +++ b/exp/lighthorizon/index/types/bitmap.go @@ -137,7 +137,7 @@ func (i *BitmapIndex) iterate(f func(index uint32)) error { for { var err error - curr, err = i.nextActive(curr + 1) + curr, err = i.nextActiveBit(curr + 1) if err != nil { if err == io.EOF { break @@ -166,16 +166,16 @@ func (i *BitmapIndex) Merge(other *BitmapIndex) error { return err } -// NextActive returns the next checkpoint (inclusive) where this index is +// NextActiveBit returns the next checkpoint (inclusive) where this index is // active. "Inclusive" means that if the index is active at `checkpoint`, this // returns `checkpoint`. -func (i *BitmapIndex) NextActive(index uint32) (uint32, error) { +func (i *BitmapIndex) NextActiveBit(index uint32) (uint32, error) { i.mutex.RLock() defer i.mutex.RUnlock() - return i.nextActive(index) + return i.nextActiveBit(index) } -func (i *BitmapIndex) nextActive(index uint32) (uint32, error) { +func (i *BitmapIndex) nextActiveBit(index uint32) (uint32, error) { if i.firstBit == 0 || index > i.lastBit { // We're past the end. // TODO: Should this be an error? or how should we signal NONE here? diff --git a/exp/lighthorizon/index/types/bitmap_test.go b/exp/lighthorizon/index/types/bitmap_test.go index d47ec58237..1c8d329879 100644 --- a/exp/lighthorizon/index/types/bitmap_test.go +++ b/exp/lighthorizon/index/types/bitmap_test.go @@ -131,7 +131,7 @@ func TestNextActive(t *testing.T) { t.Run("empty", func(t *testing.T) { index := &BitmapIndex{} - i, err := index.NextActive(0) + i, err := index.NextActiveBit(0) assert.Equal(t, uint32(0), i) assert.EqualError(t, err, io.EOF.Error()) }) @@ -142,7 +142,7 @@ func TestNextActive(t *testing.T) { index.SetActive(3) // 16 is well-past the end - i, err := index.NextActive(16) + i, err := index.NextActiveBit(16) assert.Equal(t, uint32(0), i) assert.EqualError(t, err, io.EOF.Error()) }) @@ -151,7 +151,7 @@ func TestNextActive(t *testing.T) { index := &BitmapIndex{} index.SetActive(1) - i, err := index.NextActive(1) + i, err := index.NextActiveBit(1) assert.NoError(t, err) assert.Equal(t, uint32(1), i) }) @@ -160,7 +160,7 @@ func TestNextActive(t *testing.T) { index := &BitmapIndex{} index.SetActive(9) - i, err := index.NextActive(1) + i, err := index.NextActiveBit(1) assert.NoError(t, err) assert.Equal(t, uint32(9), i) }) @@ -170,19 +170,19 @@ func TestNextActive(t *testing.T) { severalSet.SetActive(11) t.Run("several bits set (first)", func(t *testing.T) { - i, err := severalSet.NextActive(9) + i, err := severalSet.NextActiveBit(9) assert.NoError(t, err) assert.Equal(t, uint32(9), i) }) t.Run("several bits set (second)", func(t *testing.T) { - i, err := severalSet.NextActive(10) + i, err := severalSet.NextActiveBit(10) assert.NoError(t, err) assert.Equal(t, uint32(11), i) }) t.Run("several bits set (second, inclusive)", func(t *testing.T) { - i, err := severalSet.NextActive(11) + i, err := severalSet.NextActiveBit(11) assert.NoError(t, err) assert.Equal(t, uint32(11), i) }) @@ -194,27 +194,27 @@ func TestNextActive(t *testing.T) { index.SetActive(129) // Before the first - i, err := index.NextActive(8) + i, err := index.NextActiveBit(8) assert.NoError(t, err) assert.Equal(t, uint32(9), i) // at the first - i, err = index.NextActive(9) + i, err = index.NextActiveBit(9) assert.NoError(t, err) assert.Equal(t, uint32(9), i) // In the middle - i, err = index.NextActive(11) + i, err = index.NextActiveBit(11) assert.NoError(t, err) assert.Equal(t, uint32(129), i) // At the end - i, err = index.NextActive(129) + i, err = index.NextActiveBit(129) assert.NoError(t, err) assert.Equal(t, uint32(129), i) // after the end - i, err = index.NextActive(130) + i, err = index.NextActiveBit(130) assert.EqualError(t, err, io.EOF.Error()) assert.Equal(t, uint32(0), i) }) From cf37fe7701b8befa7dd5df7a939db7e7f5b3ebd2 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Mon, 8 Aug 2022 09:42:23 -0700 Subject: [PATCH 3/3] More checkpoint -> bitmap renames --- exp/lighthorizon/index/backend/gzip.go | 2 +- exp/lighthorizon/index/types/bitmap.go | 38 ++++++++++----------- exp/lighthorizon/index/types/bitmap_test.go | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/exp/lighthorizon/index/backend/gzip.go b/exp/lighthorizon/index/backend/gzip.go index e285564c8d..63c8e332c2 100644 --- a/exp/lighthorizon/index/backend/gzip.go +++ b/exp/lighthorizon/index/backend/gzip.go @@ -53,7 +53,7 @@ func readGzippedFrom(r io.Reader) (types.NamedIndices, int64, error) { return nil, n, err } - ind, err := types.NewCheckpointIndex(buf.Bytes()) + ind, err := types.NewBitmapIndex(buf.Bytes()) if err != nil { return nil, n, err } diff --git a/exp/lighthorizon/index/types/bitmap.go b/exp/lighthorizon/index/types/bitmap.go index d250f92322..256f8c8d35 100644 --- a/exp/lighthorizon/index/types/bitmap.go +++ b/exp/lighthorizon/index/types/bitmap.go @@ -8,7 +8,7 @@ import ( "github.com/stellar/go/xdr" ) -const CheckpointIndexVersion = 1 +const BitmapIndexVersion = 1 type BitmapIndex struct { mutex sync.RWMutex @@ -19,17 +19,17 @@ type BitmapIndex struct { type NamedIndices map[string]*BitmapIndex -func NewCheckpointIndex(b []byte) (*BitmapIndex, error) { - xdrCheckpoint := xdr.BitmapIndex{} - err := xdrCheckpoint.UnmarshalBinary(b) +func NewBitmapIndex(b []byte) (*BitmapIndex, error) { + xdrBitmap := xdr.BitmapIndex{} + err := xdrBitmap.UnmarshalBinary(b) if err != nil { return nil, err } - return NewCheckpointIndexFromXDR(xdrCheckpoint), nil + return NewBitmapIndexFromXDR(xdrBitmap), nil } -func NewCheckpointIndexFromXDR(index xdr.BitmapIndex) *BitmapIndex { +func NewBitmapIndexFromXDR(index xdr.BitmapIndex) *BitmapIndex { return &BitmapIndex{ bitmap: index.Bitmap[:], firstBit: uint32(index.FirstBit), @@ -166,31 +166,31 @@ func (i *BitmapIndex) Merge(other *BitmapIndex) error { return err } -// NextActiveBit returns the next checkpoint (inclusive) where this index is -// active. "Inclusive" means that if the index is active at `checkpoint`, this -// returns `checkpoint`. -func (i *BitmapIndex) NextActiveBit(index uint32) (uint32, error) { +// NextActiveBit returns the next bit position (inclusive) where this index is +// active. "Inclusive" means that if it's already active at `position`, this +// returns `position`. +func (i *BitmapIndex) NextActiveBit(position uint32) (uint32, error) { i.mutex.RLock() defer i.mutex.RUnlock() - return i.nextActiveBit(index) + return i.nextActiveBit(position) } -func (i *BitmapIndex) nextActiveBit(index uint32) (uint32, error) { - if i.firstBit == 0 || index > i.lastBit { +func (i *BitmapIndex) nextActiveBit(position uint32) (uint32, error) { + if i.firstBit == 0 || position > i.lastBit { // We're past the end. // TODO: Should this be an error? or how should we signal NONE here? return 0, io.EOF } - if index < i.firstBit { - index = i.firstBit + if position < i.firstBit { + position = i.firstBit } // Must be within the range, find the first non-zero after our start - loc := (index - i.rangeFirstBit()) / 8 + loc := (position - i.rangeFirstBit()) / 8 // Is it in the same byte? - if shift, ok := maxBitAfter(i.bitmap[loc], (index-1)%8); ok { + if shift, ok := maxBitAfter(i.bitmap[loc], (position-1)%8); ok { return i.rangeFirstBit() + (loc * 8) + shift, nil } @@ -238,8 +238,8 @@ func (i *BitmapIndex) Buffer() *bytes.Buffer { i.mutex.RLock() defer i.mutex.RUnlock() - xdrCheckpoint := i.ToXDR() - b, err := xdrCheckpoint.MarshalBinary() + xdrBitmap := i.ToXDR() + b, err := xdrBitmap.MarshalBinary() if err != nil { panic(err) } diff --git a/exp/lighthorizon/index/types/bitmap_test.go b/exp/lighthorizon/index/types/bitmap_test.go index 1c8d329879..7d5848f67a 100644 --- a/exp/lighthorizon/index/types/bitmap_test.go +++ b/exp/lighthorizon/index/types/bitmap_test.go @@ -15,7 +15,7 @@ func TestNewFromBytes(t *testing.T) { index := &BitmapIndex{} index.SetActive(i) b := index.Flush() - newIndex, err := NewCheckpointIndex(b) + newIndex, err := NewBitmapIndex(b) require.NoError(t, err) assert.Equal(t, index.firstBit, newIndex.firstBit) assert.Equal(t, index.lastBit, newIndex.lastBit)