-
Notifications
You must be signed in to change notification settings - Fork 269
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement the
lazy_set
feature (#750)
- Loading branch information
1 parent
6e98074
commit f706bde
Showing
20 changed files
with
1,237 additions
and
312 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
# LegacyDump | ||
|
||
`legacydump` is a command line tool to generate a `iavl` tree based on the legacy format of the node key. | ||
This tool is used for testing the `lazy loading and set` feature of the `iavl` tree. | ||
|
||
## Usage | ||
|
||
It takes 5 arguments: | ||
|
||
- dbtype: the type of database to use. | ||
- dbdir: the directory to store the database. | ||
- `random` or `sequential`: The `sequential` option will generate the tree from `1` to `version` in order and delete versions from `1` to `removal version`. The `random` option will delete `removal version` versions randomly. | ||
- version: the upto number of versions to generate. | ||
- removal version: the number of versions to remove. | ||
|
||
```shell | ||
go run . <dbtype> <dbdir> <random|sequential> <version> <removal version> | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
module github.com/cosmos/iavl/cmd/legacydump | ||
|
||
go 1.20 | ||
|
||
require ( | ||
github.com/cometbft/cometbft-db v0.7.0 | ||
github.com/cosmos/iavl v0.20.0 | ||
) | ||
|
||
require ( | ||
github.com/cespare/xxhash v1.1.0 // indirect | ||
github.com/confio/ics23/go v0.9.0 // indirect | ||
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect | ||
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect | ||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect | ||
github.com/dustin/go-humanize v1.0.0 // indirect | ||
github.com/gogo/protobuf v1.3.2 // indirect | ||
github.com/golang/protobuf v1.5.2 // indirect | ||
github.com/golang/snappy v0.0.4 // indirect | ||
github.com/google/btree v1.1.2 // indirect | ||
github.com/jmhodges/levigo v1.0.0 // indirect | ||
github.com/klauspost/compress v1.15.9 // indirect | ||
github.com/pkg/errors v0.9.1 // indirect | ||
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect | ||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect | ||
go.etcd.io/bbolt v1.3.6 // indirect | ||
golang.org/x/crypto v0.4.0 // indirect | ||
golang.org/x/net v0.4.0 // indirect | ||
golang.org/x/sys v0.6.0 // indirect | ||
google.golang.org/protobuf v1.28.1 // indirect | ||
) |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
package main | ||
|
||
import ( | ||
cryptorand "crypto/rand" | ||
"fmt" | ||
"math/rand" | ||
"os" | ||
"path/filepath" | ||
"strconv" | ||
|
||
dbm "github.com/cometbft/cometbft-db" | ||
"github.com/cosmos/iavl" | ||
) | ||
|
||
const ( | ||
DefaultCacheSize = 1000 | ||
) | ||
|
||
func main() { | ||
args := os.Args[1:] | ||
if len(args) < 5 { | ||
fmt.Fprintln(os.Stderr, "Usage: legacydump <dbtype> <dbdir> <random|sequential> <version> <removal version>") | ||
os.Exit(1) | ||
} | ||
|
||
version, err := strconv.Atoi(args[3]) | ||
if err != nil { | ||
fmt.Fprintf(os.Stderr, "Invalid version number: %s\n", err) | ||
os.Exit(1) | ||
} | ||
|
||
removalVersion, err := strconv.Atoi(args[4]) | ||
if err != nil { | ||
fmt.Fprintf(os.Stderr, "Invalid removal version number: %s\n", err) | ||
} | ||
|
||
if err = GenerateTree(args[0], args[1], args[2], version, removalVersion); err != nil { | ||
fmt.Fprintf(os.Stderr, "Error generating tree: %s\n", err) | ||
} | ||
} | ||
|
||
func openDB(dbType, dbDir string) (dbm.DB, error) { | ||
dir, err := filepath.Abs(dbDir) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
db, err := dbm.NewDB("test", dbm.BackendType(dbType), dir) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return db, nil | ||
} | ||
|
||
// GenerateTree generates a tree with the given number of versions. | ||
func GenerateTree(dbType, dbDir, mode string, version, removalVersion int) error { | ||
db, err := openDB(dbType, dbDir) | ||
if err != nil { | ||
return err | ||
} | ||
defer db.Close() | ||
|
||
switch mode { | ||
case "random": | ||
return generateRandomTree(db, version, removalVersion) | ||
case "sequential": | ||
_, err = generateSequentialTree(db, version, removalVersion) | ||
return err | ||
default: | ||
return fmt.Errorf("invalid mode: %s", mode) | ||
} | ||
} | ||
|
||
func generateRandomTree(db dbm.DB, version, removalVersion int) error { | ||
t, err := generateSequentialTree(db, version, 0) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
// delete the versions | ||
versions := make([]int64, version) | ||
for i := 0; i < version; i++ { | ||
versions[i] = int64(i + 1) | ||
} | ||
|
||
// make sure the latest version is not deleted | ||
for i := 1; i <= removalVersion; i++ { | ||
index := rand.Intn(version - i) | ||
if err := t.DeleteVersion(versions[index]); err != nil { | ||
return err | ||
} | ||
versions[index], versions[version-i-1] = versions[version-i-1], versions[index] | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func generateSequentialTree(db dbm.DB, version, removalVersion int) (*iavl.MutableTree, error) { | ||
t, err := iavl.NewMutableTreeWithOpts(db, DefaultCacheSize, nil, false) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
for i := 0; i < version; i++ { | ||
leafCount := rand.Int31n(50) | ||
for j := int32(0); j < leafCount; j++ { | ||
t.Set(randBytes(32), randBytes(32)) | ||
} | ||
if _, _, err = t.SaveVersion(); err != nil { | ||
return nil, err | ||
} | ||
} | ||
|
||
if removalVersion > 0 { | ||
err = t.DeleteVersionsRange(1, int64(removalVersion)+1) | ||
} | ||
|
||
return t, err | ||
} | ||
|
||
func randBytes(length int) []byte { | ||
key := make([]byte, length) | ||
_, _ = cryptorand.Read(key) | ||
return key | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.