trie: path based scheme implementing (#598)

* core/state: move account definition to core/types

Reference: ethereum/go-ethereum#27323

* trie: add path base utils

* triedb: implement history and adding some test utils

* trie/triedb/pathdb: implement difflayer and disklayer

* Fix some issues related to history, and add logic checking maxbyte when is zero for retrieving ancient ranges with maxbyte is zero

* trie/triedb/pathdb: implement database.go

* freezer: Add unit test and docs for support freezer reading with no limit size

* trie/triedb/pathdb: add database and difflayer tests

* triedb/pathdb: implement journal and add more comments

---------

Co-authored-by: Huy Ngo <[email protected]>

cmd/ronin/snapshot.go

@@ -496,14 +496,14 @@ func dumpState(ctx *cli.Context) error {
 		Root common.Hash `json:"root"`
 	}{root})
 	for accIt.Next() {
-		account, err := snapshot.FullAccount(accIt.Account())
+		account, err := types.FullAccount(accIt.Account())
 		if err != nil {
 			return err
 		}
 		da := &state.DumpAccount{
 			Balance:   account.Balance.String(),
 			Nonce:     account.Nonce,
-			Root:      account.Root,
+			Root:      account.Root.Bytes(),
 			CodeHash:  account.CodeHash,
 			SecureKey: accIt.Hash().Bytes(),
 		}

common/types.go

@@ -79,6 +79,11 @@ func (h Hash) Big() *big.Int { return new(big.Int).SetBytes(h[:]) }
 // Hex converts a hash to a hex string.
 func (h Hash) Hex() string { return hexutil.Encode(h[:]) }
 
+// Cmp compares two hashes.
+func (h Hash) Cmp(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
 // TerminalString implements log.TerminalStringer, formatting a string for console
 // output during logging.
 func (h Hash) TerminalString() string {

core/rawdb/accessors_state.go

@@ -73,7 +73,7 @@ func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
 /* Function below support Path base state trie scheme */
 
 // ReadStateId retrieves the state id with the provided state root. (Return pointer can detect that Statid is valid or not, nil is invalid)
-func ReadStateId(db ethdb.KeyValueReader, root common.Hash) *uint64 {
+func ReadStateID(db ethdb.KeyValueReader, root common.Hash) *uint64 {
 	data, err := db.Get(stateIDKey(root))
 	if err != nil || len(data) == 0 {
 		return nil

core/rawdb/accessors_trie.go

@@ -65,7 +65,7 @@ func (h *hasher) release() {
 }
 
 // ReadAccountTrieNode retrieves the account trie node and the associated node
-// hash with the specified node path.
+// hash with the specified node path. If it's empty, return empty hash.
 func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) {
 	data, err := db.Get(accountTrieNodeKey(path))
 	if err != nil {

core/rawdb/freezer.go

@@ -194,9 +194,11 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) {
 
 // AncientRange retrieves multiple items in sequence, starting from the index 'start'.
 // It will return
-//   - at most 'max' items,
-//   - at least 1 item (even if exceeding the maxByteSize), but will otherwise
-//     return as many items as fit into maxByteSize.
+//   - at most 'count' items,
+//   - if maxBytes is specified: at least 1 item (even if exceeding the maxByteSize),
+//     but will otherwise return as many items as fit into maxByteSize.
+//   - if maxBytes is not specified, 'count' items will be returned if they are present.Retru
+//   - if maxBytes is not specified, 'count' items will be returned if they are present.Retru 
 func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
 	if table := f.tables[kind]; table != nil {
 		return table.RetrieveItems(start, count, maxBytes)

core/rawdb/freezer_resettable.go

@@ -206,8 +206,9 @@ func (f *ResettableFreezer) Sync() error {
 func cleanup(pathToDelete string) error {
 	parentDir := filepath.Dir(pathToDelete)
 
-	if _, err := os.Lstat(parentDir); err != nil {
-		return err
+	// In case Parent directory does not exist, return nil, no need to cleanup.
+	if _, err := os.Lstat(parentDir); os.IsNotExist(err) {
+		return nil
 	}
 	dir, err := os.Open(parentDir)
 	if err != nil {

core/rawdb/freezer_table.go

@@ -679,7 +679,7 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e
 		if !t.noCompression {
 			decompressedSize, _ = snappy.DecodedLen(item)
 		}
-		if i > 0 && uint64(outputSize+decompressedSize) > maxBytes {
+		if i > 0 && maxBytes != 0 && uint64(outputSize+decompressedSize) > maxBytes {
 			break
 		}
 		if !t.noCompression {
@@ -697,14 +697,16 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e
 }
 
 // retrieveItems reads up to 'count' items from the table. It reads at least
-// one item, but otherwise avoids reading more than maxBytes bytes.
-// It returns the (potentially compressed) data, and the sizes.
+// one item, but otherwise avoids reading more than maxBytes bytes. Freezer
+// will ignore the size limitation and continuously allocate memory to store
+// data if maxBytes is 0. It returns the (potentially compressed) data, and
+// the sizes.
 func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) {
 	t.lock.RLock()
 	defer t.lock.RUnlock()
 
 	// Ensure the table and the item is accessible
-	if t.index == nil || t.head == nil {
+	if t.index == nil || t.head == nil || t.meta == nil {
 		return nil, nil, errClosed
 	}
 	items := t.items.Load() // max number
@@ -714,35 +716,39 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i
 	if items <= start || hidden > start || count == 0 {
 		return nil, nil, errOutOfBounds
 	}
+
 	if start+count > items {
 		count = items - start
 	}
-	var (
-		output     = make([]byte, maxBytes) // Buffer to read data into
-		outputSize int                      // Used size of that buffer
-	)
+
+	var output []byte // Buffer to read data into
+
+	if maxBytes != 0 {
+		output = make([]byte, 0, maxBytes)
+	} else {
+		output = make([]byte, 0, 1024) // initial buffer cap
+	}
+
 	// readData is a helper method to read a single data item from disk.
 	readData := func(fileId, start uint32, length int) error {
 		// In case a small limit is used, and the elements are large, may need to
 		// realloc the read-buffer when reading the first (and only) item.
-		if len(output) < length {
-			output = make([]byte, length)
-		}
+		output = grow(output, length)
 		dataFile, exist := t.files[fileId]
 		if !exist {
 			return fmt.Errorf("missing data file %d", fileId)
 		}
-		if _, err := dataFile.ReadAt(output[outputSize:outputSize+length], int64(start)); err != nil {
-			return err
+		if _, err := dataFile.ReadAt(output[len(output)-length:], int64(start)); err != nil {
+			return fmt.Errorf("%w, fileid: %d, start: %d, length: %d", err, fileId, start, length)
 		}
-		outputSize += length
 		return nil
 	}
 	// Read all the indexes in one go
 	indices, err := t.getIndices(start, count)
 	if err != nil {
 		return nil, nil, err
 	}
+
 	var (
 		sizes      []int               // The sizes for each element
 		totalSize  = 0                 // The total size of all data read so far
@@ -766,7 +772,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i
 			}
 			readStart = 0
 		}
-		if i > 0 && uint64(totalSize+size) > maxBytes {
+		if i > 0 && uint64(totalSize+size) > maxBytes && maxBytes != 0 {
 			// About to break out due to byte limit being exceeded. We don't
 			// read this last item, but we need to do the deferred reads now.
 			if unreadSize > 0 {
@@ -780,15 +786,17 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i
 		unreadSize += size
 		totalSize += size
 		sizes = append(sizes, size)
-		if i == len(indices)-2 || uint64(totalSize) > maxBytes {
+		if i == len(indices)-2 || (uint64(totalSize) > maxBytes && maxBytes != 0) {
 			// Last item, need to do the read now
 			if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil {
 				return nil, nil, err
 			}
 			break
 		}
 	}
-	return output[:outputSize], sizes, nil
+	// Update metrics.
+	t.readMeter.Mark(int64(totalSize))
+	return output, sizes, nil
 }
 
 // has returns an indicator whether the specified number data

core/rawdb/freezer_table_test.go

@@ -829,3 +829,49 @@ func TestSequentialReadByteLimit(t *testing.T) {
 		}
 	}
 }
+
+// TestSequentialReadNoByteLimit tests the batch-read if maxBytes is not specified.
+// Freezer should return the requested items regardless the size limitation.
+func TestSequentialReadNoByteLimit(t *testing.T) {
+	rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
+	fname := fmt.Sprintf("batchread-3-%d", rand.Uint64())
+	{ // Fill table
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Write 10 bytes 30 times,
+		// Splitting it at every 100 bytes (10 items)
+		writeChunks(t, f, 30, 10)
+		f.Close()
+	}
+	for i, tc := range []struct {
+		items uint64
+		want  int
+	}{
+		{1, 1},
+		{30, 30},
+		{31, 30},
+	} {
+		{
+			f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			items, err := f.RetrieveItems(0, tc.items, 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if have, want := len(items), tc.want; have != want {
+				t.Fatalf("test %d: want %d items, have %d ", i, want, have)
+			}
+			for ii, have := range items {
+				want := getChunk(10, ii)
+				if !bytes.Equal(want, have) {
+					t.Fatalf("test %d: data corruption item %d: have\n%x\n, want \n%x\n", i, ii, have, want)
+				}
+			}
+			f.Close()
+		}
+	}
+}

core/rawdb/freezer_utils.go

@@ -123,3 +123,19 @@ func truncateFreezerFile(file *os.File, size int64) error {
 	}
 	return nil
 }
+
+// grow prepares the slice space for new item, and doubles the slice capacity
+// if space is not enough.
+func grow(buf []byte, n int) []byte {
+	if cap(buf)-len(buf) < n {
+		newcap := 2 * cap(buf)
+		if newcap-len(buf) < n {
+			newcap = len(buf) + n
+		}
+		nbuf := make([]byte, len(buf), newcap)
+		copy(nbuf, buf)
+		buf = nbuf
+	}
+	buf = buf[:len(buf)+n]
+	return buf
+}

core/state/snapshot/conversion.go

@@ -17,7 +17,6 @@
 package snapshot
 
 import (
-	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -28,6 +27,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -300,7 +300,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou
 				fullData []byte
 			)
 			if leafCallback == nil {
-				fullData, err = FullAccountRLP(it.(AccountIterator).Account())
+				fullData, err = types.FullAccountRLP(it.(AccountIterator).Account())
 				if err != nil {
 					return stop(err)
 				}
@@ -312,7 +312,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou
 					return stop(err)
 				}
 				// Fetch the next account and process it concurrently
-				account, err := FullAccount(it.(AccountIterator).Account())
+				account, err := types.FullAccount(it.(AccountIterator).Account())
 				if err != nil {
 					return stop(err)
 				}
@@ -322,7 +322,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou
 						results <- err
 						return
 					}
-					if !bytes.Equal(account.Root, subroot.Bytes()) {
+					if account.Root != subroot {
 						results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot)
 						return
 					}