Skip to content

Commit

Permalink
rm block header index (#1421)
Browse files Browse the repository at this point in the history 
* rm block index

* rm unused code

* rm unused code
  • Loading branch information
@hero5512
hero5512 authored Jan 3, 2023
1 parent 7f7b1c5 commit 7eea502
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 148 deletions.
44 changes: 0 additions & 44 deletions core/store/ledgerstore/block_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,50 +295,6 @@ func (this *BlockStore) SaveCurrentBlock(height uint32, blockHash common.Uint256
return nil
}

//GetHeaderIndexList return the head index store in header index list
func (this *BlockStore) GetHeaderIndexList() (map[uint32]common.Uint256, error) {
result := make(map[uint32]common.Uint256)
iter := this.store.NewIterator([]byte{byte(scom.IX_HEADER_HASH_LIST)})
defer iter.Release()
for iter.Next() {
startCount, err := genStartHeightByHeaderIndexKey(iter.Key())
if err != nil {
return nil, fmt.Errorf("genStartHeightByHeaderIndexKey error %s", err)
}
reader := bytes.NewReader(iter.Value())
count, err := serialization.ReadUint32(reader)
if err != nil {
return nil, fmt.Errorf("serialization.ReadUint32 count error %s", err)
}
for i := uint32(0); i < count; i++ {
height := startCount + i
blockHash := common.Uint256{}
err = blockHash.Deserialize(reader)
if err != nil {
return nil, fmt.Errorf("blockHash.Deserialize error %s", err)
}
result[height] = blockHash
}
}
if err := iter.Error(); err != nil {
return nil, err
}
return result, nil
}

//SaveHeaderIndexList persist header index list to store
func (this *BlockStore) SaveHeaderIndexList(startIndex uint32, indexList []common.Uint256) {
indexKey := genHeaderIndexListKey(startIndex)
indexSize := uint32(len(indexList))
value := common.NewZeroCopySink(nil)
value.WriteUint32(indexSize)
for _, hash := range indexList {
value.WriteHash(hash)
}

this.store.BatchPut(indexKey, value.Bytes())
}

//GetBlockHash return block hash by block height
func (this *BlockStore) GetBlockHash(height uint32) (common.Uint256, error) {
key := genBlockHashKey(height)
Expand Down
46 changes: 0 additions & 46 deletions core/store/ledgerstore/block_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ package ledgerstore

import (
"crypto/sha256"
"fmt"
"testing"
"time"

Expand Down Expand Up @@ -184,51 +183,6 @@ func TestSaveTransaction(t *testing.T) {
}
}

func TestHeaderIndexList(t *testing.T) {
testBlockStore.NewBatch()
startHeight := uint32(0)
size := uint32(100)
indexMap := make(map[uint32]common.Uint256, size)
indexList := make([]common.Uint256, 0)
for i := startHeight; i < size; i++ {
hash := common.Uint256(sha256.Sum256([]byte(fmt.Sprintf("%v", i))))
indexMap[i] = hash
indexList = append(indexList, hash)
}
testBlockStore.SaveHeaderIndexList(startHeight, indexList)
startHeight = uint32(100)
size = uint32(100)
indexMap = make(map[uint32]common.Uint256, size)
for i := startHeight; i < size; i++ {
hash := common.Uint256(sha256.Sum256([]byte(fmt.Sprintf("%v", i))))
indexMap[i] = hash
indexList = append(indexList, hash)
}
err := testBlockStore.CommitTo()
if err != nil {
t.Errorf("CommitTo error %s", err)
return
}

totalMap, err := testBlockStore.GetHeaderIndexList()
if err != nil {
t.Errorf("GetHeaderIndexList error %s", err)
return
}

for height, hash := range indexList {
h, ok := totalMap[uint32(height)]
if !ok {
t.Errorf("TestHeaderIndexList failed height:%d hash not exist", height)
return
}
if hash != h {
t.Errorf("TestHeaderIndexList failed height:%d hash %x != %x", height, hash, h)
return
}
}
}

func TestSaveHeader(t *testing.T) {
acc1 := account.NewAccount("")
acc2 := account.NewAccount("")
Expand Down
88 changes: 88 additions & 0 deletions core/store/ledgerstore/header_Index_cache.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Copyright (C) 2018 The ontology Authors
* This file is part of The ontology library.
*
* The ontology is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* The ontology is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with The ontology. If not, see <http://www.gnu.org/licenses/>.
*/
//Header index cache
package ledgerstore

import "github.com/ontio/ontology/common"

const (
HEADER_INDEX_MAX_SIZE = uint32(2000) //Max size of saving header index to cache
)

type HeaderIndexCache struct {
headerIndex map[uint32]common.Uint256 //Header index, Mapping header height => block hash
firstIndex uint32 //First index of cache
lastIndex uint32 //Last index of cache
}

func NewHeaderIndexCache() *HeaderIndexCache {
return &HeaderIndexCache{
headerIndex: make(map[uint32]common.Uint256),
}
}

func (this *HeaderIndexCache) setHeaderIndex(curBlockHeight uint32, curHeaderHeight uint32, blockHash common.Uint256) {
this.headerIndex[curHeaderHeight] = blockHash
if this.getLastIndex() < curHeaderHeight {
this.lastIndex = curHeaderHeight
}
if this.getFirstIndex() < curBlockHeight {
cacheSize := curBlockHeight - this.getFirstIndex() + 1
for height := this.getFirstIndex(); cacheSize > HEADER_INDEX_MAX_SIZE; cacheSize-- {
this.delHeaderIndex(height)
height++
this.firstIndex = height
}
}
}

func (this *HeaderIndexCache) delHeaderIndex(height uint32) {
delete(this.headerIndex, height)
}

func (this *HeaderIndexCache) getHeaderIndex(height uint32) common.Uint256 {
blockHash, ok := this.headerIndex[height]
if !ok {
return common.Uint256{}
}
return blockHash
}

func (this *HeaderIndexCache) setLastIndex(lastIndex uint32) {
this.lastIndex = lastIndex
}

func (this *HeaderIndexCache) getLastIndex() uint32 {
return this.lastIndex
}

func (this *HeaderIndexCache) setFirstIndex(firstIndex uint32) {
this.firstIndex = firstIndex
}

func (this *HeaderIndexCache) getFirstIndex() uint32 {
return this.firstIndex
}

func (this *HeaderIndexCache) getCurrentHeaderHash() common.Uint256 {
blockHash, ok := this.headerIndex[this.getLastIndex()]
if !ok {
return common.Uint256{}
}
return blockHash
}
87 changes: 29 additions & 58 deletions core/store/ledgerstore/ledger_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,10 @@ type LedgerStoreImp struct {
stateStore *StateStore //StateStore for saving state data, like balance, smart contract execution result, and so on.
eventStore *EventStore //EventStore for saving log those gen after smart contract executed.
crossChainStore *CrossChainStore //crossChainStore for saving cross chain msg.
storedIndexCount uint32 //record the count of have saved block index
currBlockHeight uint32 //Current block height
currBlockHash common.Uint256 //Current block hash
headerCache map[common.Uint256]*types.Header //BlockHash => Header
headerIndex map[uint32]common.Uint256 //Header index, Mapping header height => block hash
headerIndexCache *HeaderIndexCache //Header index cache, Mapping header height => block hash
vbftPeerInfoMap map[uint32]map[string]uint32 //key:block height,value:peerInfo
lock sync.RWMutex
stateHashCheckHeight uint32
Expand All @@ -109,7 +108,6 @@ type LedgerStoreImp struct {
//NewLedgerStore return LedgerStoreImp instance
func NewLedgerStore(dataDir string, stateHashHeight uint32) (*LedgerStoreImp, error) {
ledgerStore := &LedgerStoreImp{
headerIndex: make(map[uint32]common.Uint256),
headerCache: make(map[common.Uint256]*types.Header, 0),
vbftPeerInfoMap: make(map[uint32]map[string]uint32),
savingBlockSemaphore: make(chan bool, 1),
Expand Down Expand Up @@ -142,6 +140,7 @@ func NewLedgerStore(dataDir string, stateHashHeight uint32) (*LedgerStoreImp, er
}
ledgerStore.eventStore = eventState

ledgerStore.headerIndexCache = NewHeaderIndexCache()
return ledgerStore, nil
}

Expand Down Expand Up @@ -242,7 +241,7 @@ func (this *LedgerStoreImp) InitLedgerStoreWithGenesisBlock(genesisBlock *types.
this.vbftPeerInfoMap[chainConfigHeight] = vbftPeerInfo
this.lock.Unlock()
val, _ := json.Marshal(vbftPeerInfo)
log.Infof("loading vbftPeerInfo at height: %s : %s", header.Height, string(val))
log.Infof("loading vbftPeerInfo at height: %v : %s", header.Height, string(val))
}
// check and fix imcompatible states
err = this.stateStore.CheckStorage()
Expand Down Expand Up @@ -294,24 +293,21 @@ func (this *LedgerStoreImp) loadCurrentBlock() error {

func (this *LedgerStoreImp) loadHeaderIndexList() error {
currBlockHeight := this.GetCurrentBlockHeight()
headerIndex, err := this.blockStore.GetHeaderIndexList()
if err != nil {
return fmt.Errorf("LoadHeaderIndexList error %s", err)
var height uint32
if currBlockHeight+1 > HEADER_INDEX_MAX_SIZE {
height = currBlockHeight - HEADER_INDEX_MAX_SIZE + 1
}
storeIndexCount := uint32(len(headerIndex))
this.headerIndex = headerIndex
this.storedIndexCount = storeIndexCount

for i := storeIndexCount; i <= currBlockHeight; i++ {
height := i
blockHash, err := this.blockStore.GetBlockHash(height)
this.headerIndexCache.setFirstIndex(height)
this.headerIndexCache.setLastIndex(height)
for i := height; i <= currBlockHeight; i++ {
blockHash, err := this.blockStore.GetBlockHash(i)
if err != nil {
return fmt.Errorf("LoadBlockHash height %d error %s", height, err)
return fmt.Errorf("LoadBlockHash height %d error %s", i, err)
}
if blockHash == common.UINT256_EMPTY {
return fmt.Errorf("LoadBlockHash height %d hash nil", height)
return fmt.Errorf("LoadBlockHash height %d hash nil", i)
}
this.headerIndex[height] = blockHash
this.headerIndexCache.setHeaderIndex(currBlockHeight, i, blockHash)
}
return nil
}
Expand Down Expand Up @@ -358,15 +354,18 @@ func (this *LedgerStoreImp) recoverStore() error {
func (this *LedgerStoreImp) setHeaderIndex(height uint32, blockHash common.Uint256) {
this.lock.Lock()
defer this.lock.Unlock()
this.headerIndex[height] = blockHash
this.headerIndexCache.setHeaderIndex(this.currBlockHeight, height, blockHash)
}

func (this *LedgerStoreImp) getHeaderIndex(height uint32) common.Uint256 {
this.lock.RLock()
defer this.lock.RUnlock()
blockHash, ok := this.headerIndex[height]
if !ok {
return common.Uint256{}
var blockHash common.Uint256
if blockHash = this.headerIndexCache.getHeaderIndex(height); blockHash == common.UINT256_EMPTY {
var err error
if blockHash, err = this.blockStore.GetBlockHash(height); err != nil {
return common.Uint256{}
}
}
return blockHash
}
Expand All @@ -376,22 +375,22 @@ func (this *LedgerStoreImp) getHeaderIndex(height uint32) common.Uint256 {
func (this *LedgerStoreImp) GetCurrentHeaderHeight() uint32 {
this.lock.RLock()
defer this.lock.RUnlock()
size := len(this.headerIndex)
if size == 0 {
return 0
idx := this.headerIndexCache.getLastIndex()
if idx == 0 {
return this.currBlockHeight
}
return uint32(size) - 1
return idx
}

//GetCurrentHeaderHash return the current header hash. The current header means the latest header.
func (this *LedgerStoreImp) GetCurrentHeaderHash() common.Uint256 {
this.lock.RLock()
defer this.lock.RUnlock()
size := len(this.headerIndex)
if size == 0 {
return common.Uint256{}
var currentHeaderHash common.Uint256
if currentHeaderHash = this.headerIndexCache.getCurrentHeaderHash(); currentHeaderHash == common.UINT256_EMPTY {
return this.currBlockHash
}
return this.headerIndex[uint32(size)-1]
return currentHeaderHash
}

func (this *LedgerStoreImp) setCurrentBlock(height uint32, blockHash common.Uint256) {
Expand Down Expand Up @@ -739,11 +738,7 @@ func (this *LedgerStoreImp) saveBlockToBlockStore(block *types.Block, bloom type
blockHeight := block.Header.Height

this.setHeaderIndex(blockHeight, blockHash)
err := this.saveHeaderIndexList()
if err != nil {
return fmt.Errorf("saveHeaderIndexList error %s", err)
}
err = this.blockStore.SaveCurrentBlock(blockHeight, blockHash)
err := this.blockStore.SaveCurrentBlock(blockHeight, blockHash)
if err != nil {
return fmt.Errorf("SaveCurrentBlock error %s", err)
}
Expand Down Expand Up @@ -1123,30 +1118,6 @@ func (this *LedgerStoreImp) handleTransaction(overlay *overlaydb.OverlayDB, cach
return notify, crossStateHashes, receipt, nil
}

func (this *LedgerStoreImp) saveHeaderIndexList() error {
this.lock.RLock()
storeCount := this.storedIndexCount
currHeight := this.currBlockHeight
if currHeight-storeCount < HEADER_INDEX_BATCH_SIZE {
this.lock.RUnlock()
return nil
}

headerList := make([]common.Uint256, HEADER_INDEX_BATCH_SIZE)
for i := uint32(0); i < HEADER_INDEX_BATCH_SIZE; i++ {
height := storeCount + i
headerList[i] = this.headerIndex[height]
}
this.lock.RUnlock()

this.blockStore.SaveHeaderIndexList(storeCount, headerList)

this.lock.Lock()
this.storedIndexCount += HEADER_INDEX_BATCH_SIZE
this.lock.Unlock()
return nil
}

//IsContainBlock return whether the block is in store
func (this *LedgerStoreImp) IsContainBlock(blockHash common.Uint256) (bool, error) {
return this.blockStore.ContainBlock(blockHash)
Expand Down

0 comments on commit 7eea502

Please sign in to comment.