Skip to content

Commit

Permalink
feat: implement the lazy_set feature (#750)
Browse files Browse the repository at this point in the history
  • Loading branch information
cool-develope authored May 19, 2023
1 parent 6e98074 commit f706bde
Show file tree
Hide file tree
Showing 20 changed files with 1,237 additions and 312 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ jobs:
# integer overflow).
- name: test & coverage report creation
run: |
go test ./... -mod=readonly -timeout 5m -short -race -coverprofile=coverage.txt -covermode=atomic
go test ./... -mod=readonly -timeout 8m
GOARCH=386 go test ./... -mod=readonly -timeout 8m
go test ./... -mod=readonly -timeout 10m -short -race -coverprofile=coverage.txt -covermode=atomic
go test ./... -mod=readonly -timeout 15m
GOARCH=386 go test ./... -mod=readonly -timeout 15m
18 changes: 18 additions & 0 deletions cmd/legacydump/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# LegacyDump

`legacydump` is a command line tool to generate a `iavl` tree based on the legacy format of the node key.
This tool is used for testing the `lazy loading and set` feature of the `iavl` tree.

## Usage

It takes 5 arguments:

- dbtype: the type of database to use.
- dbdir: the directory to store the database.
- `random` or `sequential`: The `sequential` option will generate the tree from `1` to `version` in order and delete versions from `1` to `removal version`. The `random` option will delete `removal version` versions randomly.
- version: the upto number of versions to generate.
- removal version: the number of versions to remove.

```shell
go run . <dbtype> <dbdir> <random|sequential> <version> <removal version>
```
31 changes: 31 additions & 0 deletions cmd/legacydump/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
module github.com/cosmos/iavl/cmd/legacydump

go 1.20

require (
github.com/cometbft/cometbft-db v0.7.0
github.com/cosmos/iavl v0.20.0
)

require (
github.com/cespare/xxhash v1.1.0 // indirect
github.com/confio/ics23/go v0.9.0 // indirect
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/net v0.4.0 // indirect
golang.org/x/sys v0.6.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
)
182 changes: 182 additions & 0 deletions cmd/legacydump/go.sum

Large diffs are not rendered by default.

125 changes: 125 additions & 0 deletions cmd/legacydump/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package main

import (
cryptorand "crypto/rand"
"fmt"
"math/rand"
"os"
"path/filepath"
"strconv"

dbm "github.com/cometbft/cometbft-db"
"github.com/cosmos/iavl"
)

const (
DefaultCacheSize = 1000
)

func main() {
args := os.Args[1:]
if len(args) < 5 {
fmt.Fprintln(os.Stderr, "Usage: legacydump <dbtype> <dbdir> <random|sequential> <version> <removal version>")
os.Exit(1)
}

version, err := strconv.Atoi(args[3])
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid version number: %s\n", err)
os.Exit(1)
}

removalVersion, err := strconv.Atoi(args[4])
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid removal version number: %s\n", err)
}

if err = GenerateTree(args[0], args[1], args[2], version, removalVersion); err != nil {
fmt.Fprintf(os.Stderr, "Error generating tree: %s\n", err)
}
}

func openDB(dbType, dbDir string) (dbm.DB, error) {
dir, err := filepath.Abs(dbDir)
if err != nil {
return nil, err
}

db, err := dbm.NewDB("test", dbm.BackendType(dbType), dir)
if err != nil {
return nil, err
}
return db, nil
}

// GenerateTree generates a tree with the given number of versions.
func GenerateTree(dbType, dbDir, mode string, version, removalVersion int) error {
db, err := openDB(dbType, dbDir)
if err != nil {
return err
}
defer db.Close()

switch mode {
case "random":
return generateRandomTree(db, version, removalVersion)
case "sequential":
_, err = generateSequentialTree(db, version, removalVersion)
return err
default:
return fmt.Errorf("invalid mode: %s", mode)
}
}

func generateRandomTree(db dbm.DB, version, removalVersion int) error {
t, err := generateSequentialTree(db, version, 0)
if err != nil {
return err
}

// delete the versions
versions := make([]int64, version)
for i := 0; i < version; i++ {
versions[i] = int64(i + 1)
}

// make sure the latest version is not deleted
for i := 1; i <= removalVersion; i++ {
index := rand.Intn(version - i)
if err := t.DeleteVersion(versions[index]); err != nil {
return err
}
versions[index], versions[version-i-1] = versions[version-i-1], versions[index]
}

return nil
}

func generateSequentialTree(db dbm.DB, version, removalVersion int) (*iavl.MutableTree, error) {
t, err := iavl.NewMutableTreeWithOpts(db, DefaultCacheSize, nil, false)
if err != nil {
return nil, err
}

for i := 0; i < version; i++ {
leafCount := rand.Int31n(50)
for j := int32(0); j < leafCount; j++ {
t.Set(randBytes(32), randBytes(32))
}
if _, _, err = t.SaveVersion(); err != nil {
return nil, err
}
}

if removalVersion > 0 {
err = t.DeleteVersionsRange(1, int64(removalVersion)+1)
}

return t, err
}

func randBytes(length int) []byte {
key := make([]byte, length)
_, _ = cryptorand.Read(key)
return key
}
2 changes: 1 addition & 1 deletion diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type KVPairReceiver func(pair *KVPair) error
//
// The algorithm don't run in constant memory strictly, but it tried the best the only
// keep minimal intermediate states in memory.
func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot *NodeKey, root *NodeKey, receiver KVPairReceiver) error {
func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot, root []byte, receiver KVPairReceiver) error {
curIter, err := NewNodeIterator(root, ndb)
if err != nil {
return err
Expand Down
54 changes: 27 additions & 27 deletions docs/node/node.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,53 +42,53 @@ func (node *Node) writeBytes(w io.Writer) error {
if node == nil {
return errors.New("cannot write nil node")
}
cause := encoding.EncodeVarint(w, int64(node.subtreeHeight))
if cause != nil {
return fmt.Errorf("writing height, %w", cause)
err := encoding.EncodeVarint(w, int64(node.subtreeHeight))
if err != nil {
return fmt.Errorf("writing height, %w", err)
}
cause = encoding.EncodeVarint(w, node.size)
if cause != nil {
return fmt.Errorf("writing size, %w", cause)
err = encoding.EncodeVarint(w, node.size)
if err != nil {
return fmt.Errorf("writing size, %w", err)
}

// Unlike writeHashByte, key is written for inner nodes.
cause = encoding.EncodeBytes(w, node.key)
if cause != nil {
return fmt.Errorf("writing key, %w", cause)
err = encoding.EncodeBytes(w, node.key)
if err != nil {
return fmt.Errorf("writing key, %w", err)
}

if node.isLeaf() {
cause = encoding.EncodeBytes(w, node.value)
if cause != nil {
return fmt.Errorf("writing value, %w", cause)
err = encoding.EncodeBytes(w, node.value)
if err != nil {
return fmt.Errorf("writing value, %w", err)
}
} else {
cause = encoding.EncodeBytes(w, node.hash)
if cause != nil {
return fmt.Errorf("writing hash, %w", cause)
err = encoding.EncodeBytes(w, node.hash)
if err != nil {
return fmt.Errorf("writing hash, %w", err)
}
if node.leftNodeKey == nil {
return ErrLeftNodeKeyEmpty
}
cause = encoding.EncodeVarint(w, node.leftNodeKey.version)
if cause != nil {
return fmt.Errorf("writing the version of left node key, %w", cause)
err = encoding.EncodeVarint(w, node.leftNodeKey.version)
if err != nil {
return fmt.Errorf("writing the version of left node key, %w", err)
}
cause = encoding.EncodeVarint(w, int64(node.leftNodeKey.nonce))
if cause != nil {
return fmt.Errorf("writing the nonce of left node key, %w", cause)
err = encoding.EncodeVarint(w, int64(node.leftNodeKey.nonce))
if err != nil {
return fmt.Errorf("writing the nonce of left node key, %w", err)
}

if node.rightNodeKey == nil {
return ErrRightNodeKeyEmpty
}
cause = encoding.EncodeVarint(w, node.rightNodeKey.version)
if cause != nil {
return fmt.Errorf("writing the version of right node key, %w", cause)
err = encoding.EncodeVarint(w, node.rightNodeKey.version)
if err != nil {
return fmt.Errorf("writing the version of right node key, %w", err)
}
cause = encoding.EncodeVarint(w, int64(node.rightNodeKey.nonce))
if cause != nil {
return fmt.Errorf("writing the nonce of right node key, %w", cause)
err = encoding.EncodeVarint(w, int64(node.rightNodeKey.nonce))
if err != nil {
return fmt.Errorf("writing the nonce of right node key, %w", err)
}
}
return nil
Expand Down
24 changes: 12 additions & 12 deletions fastnode/fast_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ func NewNode(key []byte, value []byte, version int64) *Node {

// DeserializeNode constructs an *FastNode from an encoded byte slice.
func DeserializeNode(key []byte, buf []byte) (*Node, error) {
ver, n, cause := encoding.DecodeVarint(buf)
if cause != nil {
return nil, fmt.Errorf("decoding fastnode.version, %w", cause)
ver, n, err := encoding.DecodeVarint(buf)
if err != nil {
return nil, fmt.Errorf("decoding fastnode.version, %w", err)
}
buf = buf[n:]

val, _, cause := encoding.DecodeBytes(buf)
if cause != nil {
return nil, fmt.Errorf("decoding fastnode.value, %w", cause)
val, _, err := encoding.DecodeBytes(buf)
if err != nil {
return nil, fmt.Errorf("decoding fastnode.value, %w", err)
}

fastNode := &Node{
Expand Down Expand Up @@ -73,13 +73,13 @@ func (fn *Node) WriteBytes(w io.Writer) error {
if fn == nil {
return errors.New("cannot write nil node")
}
cause := encoding.EncodeVarint(w, fn.versionLastUpdatedAt)
if cause != nil {
return fmt.Errorf("writing version last updated at, %w", cause)
err := encoding.EncodeVarint(w, fn.versionLastUpdatedAt)
if err != nil {
return fmt.Errorf("writing version last updated at, %w", err)
}
cause = encoding.EncodeBytes(w, fn.value)
if cause != nil {
return fmt.Errorf("writing value, %w", cause)
err = encoding.EncodeBytes(w, fn.value)
if err != nil {
return fmt.Errorf("writing value, %w", err)
}
return nil
}
15 changes: 8 additions & 7 deletions import.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ type Importer struct {
batch db.Batch
batchSize uint32
stack []*Node
nonces []int32
nonces []uint32
}

// newImporter creates a new Importer for an empty MutableTree.
Expand All @@ -50,7 +50,7 @@ func newImporter(tree *MutableTree, version int64) (*Importer, error) {
version: version,
batch: tree.ndb.db.NewBatch(),
stack: make([]*Node, 0, 8),
nonces: make([]int32, version+1),
nonces: make([]uint32, version+1),
}, nil
}

Expand All @@ -74,7 +74,7 @@ func (i *Importer) writeNode(node *Node) error {
bytesCopy := make([]byte, buf.Len())
copy(bytesCopy, buf.Bytes())

if err := i.batch.Set(i.tree.ndb.nodeKey(node.nodeKey), bytesCopy); err != nil {
if err := i.batch.Set(i.tree.ndb.nodeKey(node.GetKey()), bytesCopy); err != nil {
return err
}

Expand Down Expand Up @@ -138,9 +138,10 @@ func (i *Importer) Add(exportNode *ExportNode) error {

node.leftNode = leftNode
node.rightNode = rightNode
node.leftNodeKey = leftNode.nodeKey
node.rightNodeKey = rightNode.nodeKey
node.leftNodeKey = leftNode.GetKey()
node.rightNodeKey = rightNode.GetKey()
node.size = leftNode.size + rightNode.size

// Update the stack now.
if err := i.writeNode(leftNode); err != nil {
return err
Expand Down Expand Up @@ -178,7 +179,7 @@ func (i *Importer) Commit() error {

switch len(i.stack) {
case 0:
if err := i.batch.Set(i.tree.ndb.nodeKey(&NodeKey{version: i.version, nonce: 1}), []byte{}); err != nil {
if err := i.batch.Set(i.tree.ndb.nodeKey(GetRootKey(i.version)), []byte{}); err != nil {
return err
}
case 1:
Expand All @@ -187,7 +188,7 @@ func (i *Importer) Commit() error {
return err
}
if i.stack[0].nodeKey.version < i.version { // it means there is no update in the given version
if err := i.batch.Set(i.tree.ndb.nodeKey(&NodeKey{version: i.version, nonce: 1}), i.tree.ndb.nodeKey(i.stack[0].nodeKey)); err != nil {
if err := i.batch.Set(i.tree.ndb.nodeKey(GetRootKey(i.version)), i.tree.ndb.nodeKey(i.stack[0].GetKey())); err != nil {
return err
}
}
Expand Down
2 changes: 1 addition & 1 deletion iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ type NodeIterator struct {
}

// NewNodeIterator returns a new NodeIterator to traverse the tree of the root node.
func NewNodeIterator(rootKey *NodeKey, ndb *nodeDB) (*NodeIterator, error) {
func NewNodeIterator(rootKey []byte, ndb *nodeDB) (*NodeIterator, error) {
if rootKey == nil {
return &NodeIterator{
nodesToVisit: []*Node{},
Expand Down
Loading

0 comments on commit f706bde

Please sign in to comment.