Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Single point of maintenance for writing and deleting tx lookup indexes #21480

Merged
merged 1 commit into from
Sep 15, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,7 @@ func (bc *BlockChain) writeHeadBlock(block *types.Block) {
// Add the block to the canonical chain number scheme and mark as the head
batch := bc.db.NewBatch()
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
rawdb.WriteTxLookupEntries(batch, block)
rawdb.WriteTxLookupEntriesByBlock(batch, block)
rawdb.WriteHeadBlockHash(batch, block.Hash())

// If the block is better than our head or is on a different chain, force update heads
Expand Down Expand Up @@ -1217,9 +1217,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// range. In this case, all tx indices of newly imported blocks should be
// generated.
if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit {
rawdb.WriteTxLookupEntries(batch, block)
rawdb.WriteTxLookupEntriesByBlock(batch, block)
} else if rawdb.ReadTxIndexTail(bc.db) != nil {
rawdb.WriteTxLookupEntries(batch, block)
rawdb.WriteTxLookupEntriesByBlock(batch, block)
}
stats.processed++
}
Expand Down Expand Up @@ -1303,7 +1303,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// Write all the data out into the database
rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body())
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
rawdb.WriteTxLookupEntries(batch, block) // Always write tx indices for live blocks, we assume they are needed
rawdb.WriteTxLookupEntriesByBlock(batch, block) // Always write tx indices for live blocks, we assume they are needed

// Write everything belongs to the blocks into the database. So that
// we can ensure all components of body is completed(body, receipts,
Expand Down
38 changes: 20 additions & 18 deletions core/rawdb/accessors_indexes.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,25 +53,29 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
return &entry.BlockIndex
}

// WriteTxLookupEntries stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntries(db ethdb.KeyValueWriter, block *types.Block) {
number := block.Number().Bytes()
for _, tx := range block.Transactions() {
if err := db.Put(txLookupKey(tx.Hash()), number); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
// writeTxLookupEntry stores a positional metadata for a transaction,
// enabling hash based transaction and receipt lookups.
func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) {
MariusVanDerWijden marked this conversation as resolved.
Show resolved Hide resolved
if err := db.Put(txLookupKey(hash), numberBytes); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
}

// WriteTxLookupEntriesByHash is identical to WriteTxLookupEntries, but does not
// require a full types.Block as input.
func WriteTxLookupEntriesByHash(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) {
// WriteTxLookupEntries is identical to WriteTxLookupEntry, but it works on
// a list of hashes
func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) {
numberBytes := new(big.Int).SetUint64(number).Bytes()
for _, hash := range hashes {
if err := db.Put(txLookupKey(hash), numberBytes); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
writeTxLookupEntry(db, hash, numberBytes)
}
}

// WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) {
numberBytes := block.Number().Bytes()
for _, tx := range block.Transactions() {
writeTxLookupEntry(db, tx.Hash(), numberBytes)
}
}

Expand All @@ -83,11 +87,9 @@ func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) {
}

// DeleteTxLookupEntries removes all transaction lookups for a given block.
func DeleteTxLookupEntriesByHash(db ethdb.KeyValueWriter, hashes []common.Hash) {
func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) {
for _, hash := range hashes {
if err := db.Delete(txLookupKey(hash)); err != nil {
log.Crit("Failed to delete transaction lookup entry", "err", err)
}
DeleteTxLookupEntry(db, hash)
}
}

Expand Down
8 changes: 4 additions & 4 deletions core/rawdb/accessors_indexes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@ func (h *testHasher) Hash() common.Hash {
// Tests that positional lookup metadata can be stored and retrieved.
func TestLookupStorage(t *testing.T) {
tests := []struct {
name string
writeTxLookupEntries func(ethdb.Writer, *types.Block)
name string
writeTxLookupEntriesByBlock func(ethdb.Writer, *types.Block)
}{
{
"DatabaseV6",
func(db ethdb.Writer, block *types.Block) {
WriteTxLookupEntries(db, block)
WriteTxLookupEntriesByBlock(db, block)
},
},
{
Expand Down Expand Up @@ -110,7 +110,7 @@ func TestLookupStorage(t *testing.T) {
// Insert all the transactions into the database, and verify contents
WriteCanonicalHash(db, block.Hash(), block.NumberU64())
WriteBlock(db, block)
tc.writeTxLookupEntries(db, block)
tc.writeTxLookupEntriesByBlock(db, block)

for i, tx := range txs {
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
Expand Down
4 changes: 2 additions & 2 deletions core/rawdb/chain_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
// Next block available, pop it off and index it
delivery := queue.PopItem().(*blockTxHashes)
lastNum = delivery.number
WriteTxLookupEntriesByHash(batch, delivery.number, delivery.hashes)
WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
blocks++
txs += len(delivery.hashes)
// If enough data was accumulated in memory or we're at the last block, dump to disk
Expand Down Expand Up @@ -276,7 +276,7 @@ func UnindexTransactions(db ethdb.Database, from uint64, to uint64) {
// Otherwise spin up the concurrent iterator and unindexer
blocks, txs := 0, 0
for delivery := range hashesCh {
DeleteTxLookupEntriesByHash(batch, delivery.hashes)
DeleteTxLookupEntries(batch, delivery.hashes)
txs += len(delivery.hashes)
blocks++

Expand Down
2 changes: 1 addition & 1 deletion light/txpool.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ func (pool *TxPool) checkMinedTxs(ctx context.Context, hash common.Hash, number
if _, err := GetBlockReceipts(ctx, pool.odr, hash, number); err != nil { // ODR caches, ignore results
return err
}
rawdb.WriteTxLookupEntries(pool.chainDb, block)
rawdb.WriteTxLookupEntriesByBlock(pool.chainDb, block)

// Update the transaction pool's state
for _, tx := range list {
Expand Down