diff --git a/basic_test.go b/basic_test.go index 418a4d23f..13996c7dd 100644 --- a/basic_test.go +++ b/basic_test.go @@ -472,7 +472,7 @@ func TestPersistence(t *testing.T) { } // Construct some tree and save it - t1, err := NewMutableTree(db, 0) + t1, err := NewMutableTree(db, 0, false) require.NoError(t, err) for key, value := range records { t1.Set([]byte(key), []byte(value)) @@ -480,7 +480,7 @@ func TestPersistence(t *testing.T) { t1.SaveVersion() // Load a tree - t2, err := NewMutableTree(db, 0) + t2, err := NewMutableTree(db, 0, false) require.NoError(t, err) t2.Load() for key, value := range records { @@ -527,7 +527,7 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() - tree, err := NewMutableTree(db, 100) + tree, err := NewMutableTree(db, 100, false) require.NoError(t, err) hash, err := tree.Hash() require.NoError(t, err) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index ef49a7f76..f845c02d4 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -24,7 +24,7 @@ func randBytes(length int) []byte { } func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { - t, err := iavl.NewMutableTreeWithOpts(db, size, nil) + t, err := iavl.NewMutableTreeWithOpts(db, size, nil, false) require.NoError(b, err) keys := make([][]byte, size) diff --git a/benchmarks/cosmos-exim/main.go b/benchmarks/cosmos-exim/main.go index 5f9742f67..4b7ee3580 100644 --- a/benchmarks/cosmos-exim/main.go +++ b/benchmarks/cosmos-exim/main.go @@ -91,7 +91,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { if err != nil { return 0, nil, err } - tree, err := iavl.NewMutableTree(tmdb.NewPrefixDB(ldb, []byte("s/k:main/")), 0) + tree, err := iavl.NewMutableTree(tmdb.NewPrefixDB(ldb, []byte("s/k:main/")), 0, false) if err != nil { return 0, nil, err } @@ -106,7 +106,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { totalStats := Stats{} for _, name := range stores { db := tmdb.NewPrefixDB(ldb, []byte("s/k:"+name+"/")) - tree, err := iavl.NewMutableTree(db, 0) + tree, err := iavl.NewMutableTree(db, 0, false) if err != nil { return 0, nil, err } @@ -171,7 +171,7 @@ func runImport(version int64, exports map[string][]*iavl.ExportNode) error { if err != nil { return err } - newTree, err := iavl.NewMutableTree(newDB, 0) + newTree, err := iavl.NewMutableTree(newDB, 0, false) if err != nil { return err } diff --git a/cmd/iaviewer/main.go b/cmd/iaviewer/main.go index 476811598..871e512ae 100644 --- a/cmd/iaviewer/main.go +++ b/cmd/iaviewer/main.go @@ -120,7 +120,7 @@ func ReadTree(dir string, version int, prefix []byte) (*iavl.MutableTree, error) db = dbm.NewPrefixDB(db, prefix) } - tree, err := iavl.NewMutableTree(db, DefaultCacheSize) + tree, err := iavl.NewMutableTree(db, DefaultCacheSize, false) if err != nil { return nil, err } diff --git a/export_test.go b/export_test.go index bea1befd3..8b7051f2d 100644 --- a/export_test.go +++ b/export_test.go @@ -14,7 +14,7 @@ import ( // setupExportTreeBasic sets up a basic tree with a handful of // create/update/delete operations over a few versions. func setupExportTreeBasic(t require.TestingT) *ImmutableTree { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) tree.Set([]byte("x"), []byte{255}) @@ -59,7 +59,7 @@ func setupExportTreeRandom(t *testing.T) *ImmutableTree { ) r := rand.New(rand.NewSource(randSeed)) - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) var version int64 @@ -119,7 +119,7 @@ func setupExportTreeSized(t require.TestingT, treeSize int) *ImmutableTree { ) r := rand.New(rand.NewSource(randSeed)) - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) for i := 0; i < treeSize; i++ { @@ -176,7 +176,7 @@ func TestExporter(t *testing.T) { func TestExporter_Import(t *testing.T) { testcases := map[string]*ImmutableTree{ - "empty tree": NewImmutableTree(db.NewMemDB(), 0), + "empty tree": NewImmutableTree(db.NewMemDB(), 0, false), "basic tree": setupExportTreeBasic(t), } if !testing.Short() { @@ -192,7 +192,7 @@ func TestExporter_Import(t *testing.T) { exporter := tree.Export() defer exporter.Close() - newTree, err := NewMutableTree(db.NewMemDB(), 0) + newTree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := newTree.Import(tree.Version()) require.NoError(t, err) @@ -256,7 +256,7 @@ func TestExporter_Close(t *testing.T) { } func TestExporter_DeleteVersionErrors(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) tree.Set([]byte("a"), []byte{1}) diff --git a/immutable_tree.go b/immutable_tree.go index 5a59b1afd..53a79f841 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -14,28 +14,31 @@ import ( // Returned key/value byte slices must not be modified, since they may point to data located inside // IAVL which would also be modified. type ImmutableTree struct { - root *Node - ndb *nodeDB - version int64 + root *Node + ndb *nodeDB + version int64 + skipFastStorageUpgrade bool } // NewImmutableTree creates both in-memory and persistent instances -func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { +func NewImmutableTree(db dbm.DB, cacheSize int, skipFastStorageUpgrade bool) *ImmutableTree { if db == nil { // In-memory Tree. return &ImmutableTree{} } return &ImmutableTree{ // NodeDB-backed Tree. - ndb: newNodeDB(db, cacheSize, nil), + ndb: newNodeDB(db, cacheSize, nil), + skipFastStorageUpgrade: skipFastStorageUpgrade, } } // NewImmutableTreeWithOpts creates an ImmutableTree with the given options. -func NewImmutableTreeWithOpts(db dbm.DB, cacheSize int, opts *Options) *ImmutableTree { +func NewImmutableTreeWithOpts(db dbm.DB, cacheSize int, opts *Options, skipFastStorageUpgrade bool) *ImmutableTree { return &ImmutableTree{ // NodeDB-backed Tree. - ndb: newNodeDB(db, cacheSize, opts), + ndb: newNodeDB(db, cacheSize, opts), + skipFastStorageUpgrade: skipFastStorageUpgrade, } } @@ -172,36 +175,40 @@ func (t *ImmutableTree) GetWithIndex(key []byte) (int64, []byte, error) { // Get returns the value of the specified key if it exists, or nil. // The returned value must not be modified, since it may point to data stored within IAVL. // Get potentially employs a more performant strategy than GetWithIndex for retrieving the value. +// If tree.skipFastStorageUpgrade is true, this will work almost the same as GetWithIndex. func (t *ImmutableTree) Get(key []byte) ([]byte, error) { if t.root == nil { return nil, nil } - // attempt to get a FastNode directly from db/cache. - // if call fails, fall back to the original IAVL logic in place. - fastNode, err := t.ndb.GetFastNode(key) - if err != nil { - _, result, err := t.root.get(t, key) - return result, err - } - - if fastNode == nil { - // If the tree is of the latest version and fast node is not in the tree - // then the regular node is not in the tree either because fast node - // represents live state. - if t.version == t.ndb.latestVersion { - return nil, nil + if !t.skipFastStorageUpgrade { + // attempt to get a FastNode directly from db/cache. + // if call fails, fall back to the original IAVL logic in place. + fastNode, err := t.ndb.GetFastNode(key) + if err != nil { + _, result, err := t.root.get(t, key) + return result, err } - _, result, err := t.root.get(t, key) - return result, err - } + if fastNode == nil { + // If the tree is of the latest version and fast node is not in the tree + // then the regular node is not in the tree either because fast node + // represents live state. + if t.version == t.ndb.latestVersion { + return nil, nil + } - if fastNode.versionLastUpdatedAt <= t.version { - return fastNode.value, nil + _, result, err := t.root.get(t, key) + return result, err + } + + if fastNode.versionLastUpdatedAt <= t.version { + return fastNode.value, nil + } } - // Otherwise the cached node was updated later than the current tree. In this case, + // otherwise skipFastStorageUpgrade is true or + // the cached node was updated later than the current tree. In this case, // we need to use the regular stategy for reading from the current tree to avoid staleness. _, result, err := t.root.get(t, key) return result, err @@ -239,13 +246,15 @@ func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) (bool, e // Iterator returns an iterator over the immutable tree. func (t *ImmutableTree) Iterator(start, end []byte, ascending bool) (dbm.Iterator, error) { - isFastCacheEnabled, err := t.IsFastCacheEnabled() - if err != nil { - return nil, err - } + if !t.skipFastStorageUpgrade { + isFastCacheEnabled, err := t.IsFastCacheEnabled() + if err != nil { + return nil, err + } - if isFastCacheEnabled { - return NewFastIterator(start, end, ascending, t.ndb), nil + if isFastCacheEnabled { + return NewFastIterator(start, end, ascending, t.ndb), nil + } } return NewIterator(start, end, ascending, t), nil } @@ -311,6 +320,7 @@ func (t *ImmutableTree) clone() *ImmutableTree { } // nodeSize is like Size, but includes inner nodes too. +// //nolint:unused func (t *ImmutableTree) nodeSize() int { size := 0 diff --git a/import_test.go b/import_test.go index a5f494f44..5fbf7d733 100644 --- a/import_test.go +++ b/import_test.go @@ -10,7 +10,7 @@ import ( ) func ExampleImporter() { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) if err != nil { // handle err } @@ -41,7 +41,7 @@ func ExampleImporter() { exported = append(exported, node) } - newTree, err := NewMutableTree(db.NewMemDB(), 0) + newTree, err := NewMutableTree(db.NewMemDB(), 0, false) if err != nil { // handle err } @@ -63,14 +63,14 @@ func ExampleImporter() { } func TestImporter_NegativeVersion(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) _, err = tree.Import(-1) require.Error(t, err) } func TestImporter_NotEmpty(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) tree.Set([]byte("a"), []byte{1}) _, _, err = tree.SaveVersion() @@ -83,13 +83,13 @@ func TestImporter_NotEmpty(t *testing.T) { func TestImporter_NotEmptyDatabase(t *testing.T) { db := db.NewMemDB() - tree, err := NewMutableTree(db, 0) + tree, err := NewMutableTree(db, 0, false) require.NoError(t, err) tree.Set([]byte("a"), []byte{1}) _, _, err = tree.SaveVersion() require.NoError(t, err) - tree, err = NewMutableTree(db, 0) + tree, err = NewMutableTree(db, 0, false) require.NoError(t, err) _, err = tree.Load() require.NoError(t, err) @@ -99,7 +99,7 @@ func TestImporter_NotEmptyDatabase(t *testing.T) { } func TestImporter_NotEmptyUnsaved(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) tree.Set([]byte("a"), []byte{1}) @@ -126,7 +126,7 @@ func TestImporter_Add(t *testing.T) { for desc, tc := range testcases { tc := tc // appease scopelint t.Run(desc, func(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(1) require.NoError(t, err) @@ -143,7 +143,7 @@ func TestImporter_Add(t *testing.T) { } func TestImporter_Add_Closed(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(1) require.NoError(t, err) @@ -155,7 +155,7 @@ func TestImporter_Add_Closed(t *testing.T) { } func TestImporter_Close(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(1) require.NoError(t, err) @@ -172,7 +172,7 @@ func TestImporter_Close(t *testing.T) { } func TestImporter_Commit(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(1) require.NoError(t, err) @@ -188,7 +188,7 @@ func TestImporter_Commit(t *testing.T) { } func TestImporter_Commit_Closed(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(1) require.NoError(t, err) @@ -203,7 +203,7 @@ func TestImporter_Commit_Closed(t *testing.T) { } func TestImporter_Commit_Empty(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) importer, err := tree.Import(3) require.NoError(t, err) @@ -232,7 +232,7 @@ func BenchmarkImport(b *testing.B) { b.StartTimer() for n := 0; n < b.N; n++ { - newTree, err := NewMutableTree(db.NewMemDB(), 0) + newTree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(b, err) importer, err := newTree.Import(tree.Version()) require.NoError(b, err) diff --git a/iterator_test.go b/iterator_test.go index 84da66d36..4164e055a 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -55,7 +55,7 @@ func TestUnsavedFastIterator_NewIterator_NilAdditions_Failure(t *testing.T) { } t.Run("Nil additions given", func(t *testing.T) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) itr := NewUnsavedFastIterator(start, end, ascending, tree.ndb, nil, tree.unsavedFastNodeRemovals) performTest(t, itr) @@ -63,7 +63,7 @@ func TestUnsavedFastIterator_NewIterator_NilAdditions_Failure(t *testing.T) { }) t.Run("Nil removals given", func(t *testing.T) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) itr := NewUnsavedFastIterator(start, end, ascending, tree.ndb, tree.unsavedFastNodeAdditions, nil) performTest(t, itr) @@ -77,7 +77,7 @@ func TestUnsavedFastIterator_NewIterator_NilAdditions_Failure(t *testing.T) { }) t.Run("Additions and removals are nil", func(t *testing.T) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) itr := NewUnsavedFastIterator(start, end, ascending, tree.ndb, nil, nil) performTest(t, itr) @@ -247,7 +247,7 @@ func iteratorSuccessTest(t *testing.T, config *iteratorTestConfig) { } func setupIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (dbm.Iterator, [][]string) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) mirror := setupMirrorForIterator(t, config, tree) @@ -264,7 +264,7 @@ func setupIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (dbm.Itera } func setupFastIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (dbm.Iterator, [][]string) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) mirror := setupMirrorForIterator(t, config, tree) @@ -276,7 +276,7 @@ func setupFastIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (dbm.I } func setupUnsavedFastIterator(t *testing.T, config *iteratorTestConfig) (dbm.Iterator, [][]string) { - tree, err := NewMutableTree(dbm.NewMemDB(), 0) + tree, err := NewMutableTree(dbm.NewMemDB(), 0, false) require.NoError(t, err) // For unsaved fast iterator, we would like to test the state where diff --git a/mutable_tree.go b/mutable_tree.go index 4f7a0e94f..abb899edc 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -36,19 +36,20 @@ type MutableTree struct { unsavedFastNodeAdditions map[string]*FastNode // FastNodes that have not yet been saved to disk unsavedFastNodeRemovals map[string]interface{} // FastNodes that have not yet been removed from disk ndb *nodeDB + skipFastStorageUpgrade bool // If true, the tree will work like no fast storage and always not upgrade fast storage mtx sync.Mutex } // NewMutableTree returns a new tree with the specified cache size and datastore. -func NewMutableTree(db dbm.DB, cacheSize int) (*MutableTree, error) { - return NewMutableTreeWithOpts(db, cacheSize, nil) +func NewMutableTree(db dbm.DB, cacheSize int, skipFastStorageUpgrade bool) (*MutableTree, error) { + return NewMutableTreeWithOpts(db, cacheSize, nil, skipFastStorageUpgrade) } // NewMutableTreeWithOpts returns a new tree with the specified options. -func NewMutableTreeWithOpts(db dbm.DB, cacheSize int, opts *Options) (*MutableTree, error) { +func NewMutableTreeWithOpts(db dbm.DB, cacheSize int, opts *Options, skipFastStorageUpgrade bool) (*MutableTree, error) { ndb := newNodeDB(db, cacheSize, opts) - head := &ImmutableTree{ndb: ndb} + head := &ImmutableTree{ndb: ndb, skipFastStorageUpgrade: skipFastStorageUpgrade} return &MutableTree{ ImmutableTree: head, @@ -59,6 +60,7 @@ func NewMutableTreeWithOpts(db dbm.DB, cacheSize int, opts *Options) (*MutableTr unsavedFastNodeAdditions: make(map[string]*FastNode), unsavedFastNodeRemovals: make(map[string]interface{}), ndb: ndb, + skipFastStorageUpgrade: skipFastStorageUpgrade, }, nil } @@ -147,12 +149,14 @@ func (tree *MutableTree) Get(key []byte) ([]byte, error) { return nil, nil } - if fastNode, ok := tree.unsavedFastNodeAdditions[unsafeToStr(key)]; ok { - return fastNode.value, nil - } - // check if node was deleted - if _, ok := tree.unsavedFastNodeRemovals[string(key)]; ok { - return nil, nil + if !tree.skipFastStorageUpgrade { + if fastNode, ok := tree.unsavedFastNodeAdditions[unsafeToStr(key)]; ok { + return fastNode.value, nil + } + // check if node was deleted + if _, ok := tree.unsavedFastNodeRemovals[string(key)]; ok { + return nil, nil + } } return tree.ImmutableTree.Get(key) @@ -177,6 +181,10 @@ func (tree *MutableTree) Iterate(fn func(key []byte, value []byte) bool) (stoppe return false, nil } + if tree.skipFastStorageUpgrade { + return tree.ImmutableTree.Iterate(fn) + } + isFastCacheEnabled, err := tree.IsFastCacheEnabled() if err != nil { return false, err @@ -198,14 +206,17 @@ func (tree *MutableTree) Iterate(fn func(key []byte, value []byte) bool) (stoppe // Iterator returns an iterator over the mutable tree. // CONTRACT: no updates are made to the tree while an iterator is active. func (tree *MutableTree) Iterator(start, end []byte, ascending bool) (dbm.Iterator, error) { - isFastCacheEnabled, err := tree.IsFastCacheEnabled() - if err != nil { - return nil, err - } + if !tree.skipFastStorageUpgrade { + isFastCacheEnabled, err := tree.IsFastCacheEnabled() + if err != nil { + return nil, err + } - if isFastCacheEnabled { - return NewUnsavedFastIterator(start, end, ascending, tree.ndb, tree.unsavedFastNodeAdditions, tree.unsavedFastNodeRemovals), nil + if isFastCacheEnabled { + return NewUnsavedFastIterator(start, end, ascending, tree.ndb, tree.unsavedFastNodeAdditions, tree.unsavedFastNodeRemovals), nil + } } + return tree.ImmutableTree.Iterator(start, end, ascending) } @@ -215,7 +226,9 @@ func (tree *MutableTree) set(key []byte, value []byte) (orphans []*Node, updated } if tree.ImmutableTree.root == nil { - tree.addUnsavedAddition(key, NewFastNode(key, value, tree.version+1)) + if !tree.skipFastStorageUpgrade { + tree.addUnsavedAddition(key, NewFastNode(key, value, tree.version+1)) + } tree.ImmutableTree.root = NewNode(key, value, tree.version+1) return nil, updated, nil } @@ -231,7 +244,9 @@ func (tree *MutableTree) recursiveSet(node *Node, key []byte, value []byte, orph version := tree.version + 1 if node.isLeaf() { - tree.addUnsavedAddition(key, NewFastNode(key, value, version)) + if !tree.skipFastStorageUpgrade { + tree.addUnsavedAddition(key, NewFastNode(key, value, version)) + } switch bytes.Compare(key, node.key) { case -1: @@ -461,10 +476,13 @@ func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) { // no versions have been saved if the latest version is non-positive if latestVersion <= 0 { if targetVersion <= 0 { - tree.mtx.Lock() - defer tree.mtx.Unlock() - _, err := tree.enableFastStorageAndCommitIfNotEnabled() - return 0, err + if !tree.skipFastStorageUpgrade { + tree.mtx.Lock() + defer tree.mtx.Unlock() + _, err := tree.enableFastStorageAndCommitIfNotEnabled() + return 0, err + } + return 0, nil } return 0, fmt.Errorf("no versions found while trying to load %v", targetVersion) } @@ -488,8 +506,9 @@ func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) { tree.versions[targetVersion] = true iTree := &ImmutableTree{ - ndb: tree.ndb, - version: targetVersion, + ndb: tree.ndb, + version: targetVersion, + skipFastStorageUpgrade: tree.skipFastStorageUpgrade, } if len(rootHash) > 0 { // If rootHash is empty then root of tree should be nil @@ -504,9 +523,11 @@ func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) { tree.ImmutableTree = iTree tree.lastSaved = iTree.clone() - // Attempt to upgrade - if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil { - return 0, err + if !tree.skipFastStorageUpgrade { + // Attempt to upgrade + if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil { + return 0, err + } } return targetVersion, nil @@ -521,10 +542,13 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { if len(roots) == 0 { if targetVersion <= 0 { - tree.mtx.Lock() - defer tree.mtx.Unlock() - _, err := tree.enableFastStorageAndCommitIfNotEnabled() - return 0, err + if !tree.skipFastStorageUpgrade { + tree.mtx.Lock() + defer tree.mtx.Unlock() + _, err := tree.enableFastStorageAndCommitIfNotEnabled() + return 0, err + } + return 0, nil } return 0, fmt.Errorf("no versions found while trying to load %v", targetVersion) } @@ -558,8 +582,9 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { } t := &ImmutableTree{ - ndb: tree.ndb, - version: latestVersion, + ndb: tree.ndb, + version: latestVersion, + skipFastStorageUpgrade: tree.skipFastStorageUpgrade, } if len(latestRoot) != 0 { @@ -574,9 +599,11 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { tree.lastSaved = t.clone() tree.allRootLoaded = true - // Attempt to upgrade - if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil { - return 0, err + if !tree.skipFastStorageUpgrade { + // Attempt to upgrade + if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil { + return 0, err + } } return latestVersion, nil @@ -594,8 +621,10 @@ func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, return latestVersion, err } - if err := tree.enableFastStorageAndCommitLocked(); err != nil { - return latestVersion, err + if !tree.skipFastStorageUpgrade { + if err := tree.enableFastStorageAndCommitLocked(); err != nil { + return latestVersion, err + } } tree.ndb.resetLatestVersion(latestVersion) @@ -620,7 +649,7 @@ func (tree *MutableTree) IsUpgradeable() (bool, error) { if err != nil { return false, err } - return !tree.ndb.hasUpgradedToFastStorage() || shouldForce, nil + return !tree.skipFastStorageUpgrade && (!tree.ndb.hasUpgradedToFastStorage() || shouldForce), nil } // enableFastStorageAndCommitIfNotEnabled if nodeDB doesn't mark fast storage as enabled, enable it, and commit the update. @@ -717,8 +746,9 @@ func (tree *MutableTree) GetImmutable(version int64) (*ImmutableTree, error) { if len(rootHash) == 0 { tree.versions[version] = true return &ImmutableTree{ - ndb: tree.ndb, - version: version, + ndb: tree.ndb, + version: version, + skipFastStorageUpgrade: tree.skipFastStorageUpgrade, }, nil } tree.versions[version] = true @@ -728,9 +758,10 @@ func (tree *MutableTree) GetImmutable(version int64) (*ImmutableTree, error) { return nil, err } return &ImmutableTree{ - root: root, - ndb: tree.ndb, - version: version, + root: root, + ndb: tree.ndb, + version: version, + skipFastStorageUpgrade: tree.skipFastStorageUpgrade, }, nil } @@ -740,30 +771,38 @@ func (tree *MutableTree) Rollback() { if tree.version > 0 { tree.ImmutableTree = tree.lastSaved.clone() } else { - tree.ImmutableTree = &ImmutableTree{ndb: tree.ndb, version: 0} + tree.ImmutableTree = &ImmutableTree{ + ndb: tree.ndb, + version: 0, + skipFastStorageUpgrade: tree.skipFastStorageUpgrade, + } } tree.orphans = map[string]int64{} - tree.unsavedFastNodeAdditions = map[string]*FastNode{} - tree.unsavedFastNodeRemovals = map[string]interface{}{} + if !tree.skipFastStorageUpgrade { + tree.unsavedFastNodeAdditions = map[string]*FastNode{} + tree.unsavedFastNodeRemovals = map[string]interface{}{} + } } // GetVersioned gets the value at the specified key and version. The returned value must not be // modified, since it may point to data stored within IAVL. func (tree *MutableTree) GetVersioned(key []byte, version int64) ([]byte, error) { if tree.VersionExists(version) { - isFastCacheEnabled, err := tree.IsFastCacheEnabled() - if err != nil { - return nil, err - } - - if isFastCacheEnabled { - fastNode, _ := tree.ndb.GetFastNode(key) - if fastNode == nil && version == tree.ndb.latestVersion { - return nil, nil + if !tree.skipFastStorageUpgrade { + isFastCacheEnabled, err := tree.IsFastCacheEnabled() + if err != nil { + return nil, err } - if fastNode != nil && fastNode.versionLastUpdatedAt <= version { - return fastNode.value, nil + if isFastCacheEnabled { + fastNode, _ := tree.ndb.GetFastNode(key) + if fastNode == nil && version == tree.ndb.latestVersion { + return nil, nil + } + + if fastNode != nil && fastNode.versionLastUpdatedAt <= version { + return fastNode.value, nil + } } } t, err := tree.GetImmutable(version) @@ -840,8 +879,10 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { } } - if err := tree.saveFastNodeVersion(); err != nil { - return nil, version, err + if !tree.skipFastStorageUpgrade { + if err := tree.saveFastNodeVersion(); err != nil { + return nil, version, err + } } if err := tree.ndb.Commit(); err != nil { @@ -857,8 +898,10 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.ImmutableTree = tree.ImmutableTree.clone() tree.lastSaved = tree.ImmutableTree.clone() tree.orphans = map[string]int64{} - tree.unsavedFastNodeAdditions = make(map[string]*FastNode) - tree.unsavedFastNodeRemovals = make(map[string]interface{}) + if !tree.skipFastStorageUpgrade { + tree.unsavedFastNodeAdditions = make(map[string]*FastNode) + tree.unsavedFastNodeRemovals = make(map[string]interface{}) + } hash, err := tree.Hash() if err != nil { diff --git a/mutable_tree_test.go b/mutable_tree_test.go index df388806b..b2c2c8ac5 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -28,7 +28,7 @@ var ( func setupMutableTree(t *testing.T) *MutableTree { memDB := db.NewMemDB() - tree, err := NewMutableTree(memDB, 0) + tree, err := NewMutableTree(memDB, 0, false) require.NoError(t, err) return tree } @@ -181,7 +181,7 @@ func TestMutableTree_LoadVersion_Empty(t *testing.T) { func TestMutableTree_LazyLoadVersion_Empty(t *testing.T) { memDB := db.NewMemDB() - tree, err := NewMutableTree(memDB, 0) + tree, err := NewMutableTree(memDB, 0, false) require.NoError(t, err) version, err := tree.LazyLoadVersion(0) @@ -200,7 +200,7 @@ func TestMutableTree_DeleteVersionsRange(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) const maxLength = 100 const fromLength = 10 @@ -216,7 +216,7 @@ func TestMutableTree_DeleteVersionsRange(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) targetVersion, err := tree.LoadVersion(int64(maxLength)) require.NoError(err) @@ -279,7 +279,7 @@ func TestMutableTree_DeleteVersionsRange(t *testing.T) { func TestMutableTree_InitialVersion(t *testing.T) { memDB := db.NewMemDB() - tree, err := NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}) + tree, err := NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}, false) require.NoError(t, err) tree.Set([]byte("a"), []byte{0x01}) @@ -293,20 +293,20 @@ func TestMutableTree_InitialVersion(t *testing.T) { assert.EqualValues(t, 10, version) // Reloading the tree with the same initial version is fine - tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}) + tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}, false) require.NoError(t, err) version, err = tree.Load() require.NoError(t, err) assert.EqualValues(t, 10, version) // Reloading the tree with an initial version beyond the lowest should error - tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 10}) + tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 10}, false) require.NoError(t, err) _, err = tree.Load() require.Error(t, err) // Reloading the tree with a lower initial version is fine, and new versions can be produced - tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 3}) + tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 3}, false) require.NoError(t, err) version, err = tree.Load() require.NoError(t, err) @@ -331,7 +331,7 @@ func TestMutableTree_SetInitialVersion(t *testing.T) { func BenchmarkMutableTree_Set(b *testing.B) { db, err := db.NewDB("test", db.MemDBBackend, "") require.NoError(b, err) - t, err := NewMutableTree(db, 100000) + t, err := NewMutableTree(db, 100000, false) require.NoError(b, err) for i := 0; i < 1000000; i++ { t.Set(randBytes(10), []byte{}) @@ -348,7 +348,7 @@ func BenchmarkMutableTree_Set(b *testing.B) { func prepareTree(t *testing.T) *MutableTree { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 1000) + tree, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) for i := 0; i < 100; i++ { tree.Set([]byte{byte(i)}, []byte("a")) @@ -362,7 +362,7 @@ func prepareTree(t *testing.T) *MutableTree { _, ver, err = tree.SaveVersion() require.True(t, ver == 2) require.NoError(t, err) - newTree, err := NewMutableTree(mdb, 1000) + newTree, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) return newTree @@ -418,18 +418,18 @@ func TestMutableTree_DeleteVersion(t *testing.T) { func TestMutableTree_LazyLoadVersionWithEmptyTree(t *testing.T) { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 1000) + tree, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) _, v1, err := tree.SaveVersion() require.NoError(t, err) - newTree1, err := NewMutableTree(mdb, 1000) + newTree1, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) v2, err := newTree1.LazyLoadVersion(1) require.NoError(t, err) require.True(t, v1 == v2) - newTree2, err := NewMutableTree(mdb, 1000) + newTree2, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) v2, err = newTree1.LoadVersion(1) require.NoError(t, err) @@ -440,7 +440,7 @@ func TestMutableTree_LazyLoadVersionWithEmptyTree(t *testing.T) { func TestMutableTree_SetSimple(t *testing.T) { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(t, err) const testKey1 = "a" @@ -611,7 +611,7 @@ func TestMutableTree_SetRemoveSet(t *testing.T) { func TestMutableTree_FastNodeIntegration(t *testing.T) { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 1000) + tree, err := NewMutableTree(mdb, 1000, false) require.NoError(t, err) const key1 = "a" @@ -676,7 +676,7 @@ func TestMutableTree_FastNodeIntegration(t *testing.T) { require.Equal(t, len(unsavedNodeRemovals), 0) // Load - t2, err := NewMutableTree(mdb, 0) + t2, err := NewMutableTree(mdb, 0, false) require.NoError(t, err) _, err = t2.Load() @@ -745,7 +745,7 @@ func TestIterator_MutableTree_Invalid(t *testing.T) { func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) { // Setup db := db.NewMemDB() - tree, err := NewMutableTree(db, 1000) + tree, err := NewMutableTree(db, 1000, false) require.NoError(t, err) // Default version when storage key does not exist in the db @@ -776,7 +776,7 @@ func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) { func TestUpgradeStorageToFast_AlreadyUpgraded_Success(t *testing.T) { // Setup db := db.NewMemDB() - tree, err := NewMutableTree(db, 1000) + tree, err := NewMutableTree(db, 1000, false) require.NoError(t, err) // Default version when storage key does not exist in the db @@ -828,7 +828,7 @@ func TestUpgradeStorageToFast_DbErrorConstructor_Failure(t *testing.T) { dbMock.EXPECT().NewBatch().Return(nil).Times(1) dbMock.EXPECT().ReverseIterator(gomock.Any(), gomock.Any()).Return(rIterMock, nil).Times(1) - tree, err := NewMutableTree(dbMock, 0) + tree, err := NewMutableTree(dbMock, 0, false) require.Nil(t, err) require.NotNil(t, tree) @@ -863,7 +863,7 @@ func TestUpgradeStorageToFast_DbErrorEnableFastStorage_Failure(t *testing.T) { batchMock.EXPECT().Set(gomock.Any(), gomock.Any()).Return(expectedError).Times(1) - tree, err := NewMutableTree(dbMock, 0) + tree, err := NewMutableTree(dbMock, 0, false) require.Nil(t, err) require.NotNil(t, tree) @@ -904,7 +904,7 @@ func TestFastStorageReUpgradeProtection_NoForceUpgrade_Success(t *testing.T) { dbMock.EXPECT().NewBatch().Return(batchMock).Times(1) dbMock.EXPECT().ReverseIterator(gomock.Any(), gomock.Any()).Return(rIterMock, nil).Times(1) // called to get latest version - tree, err := NewMutableTree(dbMock, 0) + tree, err := NewMutableTree(dbMock, 0, false) require.Nil(t, err) require.NotNil(t, tree) @@ -997,7 +997,7 @@ func TestFastStorageReUpgradeProtection_ForceUpgradeFirstTime_NoForceSecondTime_ iterMock.EXPECT().Valid().Return(false).Times(1) iterMock.EXPECT().Close().Return(nil).Times(1) - tree, err := NewMutableTree(dbMock, 0) + tree, err := NewMutableTree(dbMock, 0, false) require.Nil(t, err) require.NotNil(t, tree) @@ -1028,7 +1028,7 @@ func TestFastStorageReUpgradeProtection_ForceUpgradeFirstTime_NoForceSecondTime_ func TestUpgradeStorageToFast_Integration_Upgraded_FastIterator_Success(t *testing.T) { // Setup - tree, mirror := setupTreeAndMirrorForUpgrade(t, 100) + tree, mirror := setupTreeAndMirror(t, 100, false) isFastCacheEnabled, err := tree.IsFastCacheEnabled() require.NoError(t, err) @@ -1048,7 +1048,7 @@ func TestUpgradeStorageToFast_Integration_Upgraded_FastIterator_Success(t *testi require.False(t, isUpgradeable) require.NoError(t, err) - sut, _ := NewMutableTree(tree.ndb.db, 1000) + sut, _ := NewMutableTree(tree.ndb.db, 1000, false) isFastCacheEnabled, err = sut.IsFastCacheEnabled() require.NoError(t, err) @@ -1095,7 +1095,7 @@ func TestUpgradeStorageToFast_Integration_Upgraded_FastIterator_Success(t *testi func TestUpgradeStorageToFast_Integration_Upgraded_GetFast_Success(t *testing.T) { // Setup - tree, mirror := setupTreeAndMirrorForUpgrade(t, 100) + tree, mirror := setupTreeAndMirror(t, 100, false) isFastCacheEnabled, err := tree.IsFastCacheEnabled() require.NoError(t, err) @@ -1115,7 +1115,7 @@ func TestUpgradeStorageToFast_Integration_Upgraded_GetFast_Success(t *testing.T) require.False(t, isUpgradeable) require.NoError(t, err) - sut, _ := NewMutableTree(tree.ndb.db, 1000) + sut, _ := NewMutableTree(tree.ndb.db, 1000, false) isFastCacheEnabled, err = sut.IsFastCacheEnabled() require.NoError(t, err) @@ -1176,7 +1176,7 @@ func TestUpgradeStorageToFast_Success(t *testing.T) { } for _, tt := range tests { - tree, mirror := setupTreeAndMirrorForUpgrade(t, tt.fields.nodeCount) + tree, mirror := setupTreeAndMirror(t, tt.fields.nodeCount, false) enabled, err := tree.enableFastStorageAndCommitIfNotEnabled() require.Nil(t, err) require.True(t, enabled) @@ -1233,7 +1233,7 @@ func TestUpgradeStorageToFast_Delete_Stale_Success(t *testing.T) { } for _, tt := range tests { - tree, mirror := setupTreeAndMirrorForUpgrade(t, tt.fields.nodeCount) + tree, mirror := setupTreeAndMirror(t, tt.fields.nodeCount, false) addStaleKey(tree.ndb, tt.fields.staleCount) enabled, err := tree.enableFastStorageAndCommitIfNotEnabled() require.Nil(t, err) @@ -1251,10 +1251,10 @@ func TestUpgradeStorageToFast_Delete_Stale_Success(t *testing.T) { } } -func setupTreeAndMirrorForUpgrade(t *testing.T, numEntries int) (*MutableTree, [][]string) { +func setupTreeAndMirror(t *testing.T, numEntries int, skipFastStorageUpgrade bool) (*MutableTree, [][]string) { db := db.NewMemDB() - tree, _ := NewMutableTree(db, 0) + tree, _ := NewMutableTree(db, 0, skipFastStorageUpgrade) var keyPrefix, valPrefix = "key", "val" @@ -1279,3 +1279,165 @@ func setupTreeAndMirrorForUpgrade(t *testing.T, numEntries int) (*MutableTree, [ }) return tree, mirror } + +func TestNoFastStorageUpgrade_Integration_SaveVersion_Load_Get_Success(t *testing.T) { + // Setup + tree, mirror := setupTreeAndMirror(t, 100, true) + + isFastCacheEnabled, err := tree.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err := tree.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + // Should Not auto enable in save version + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + isFastCacheEnabled, err = tree.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err = tree.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + sut, _ := NewMutableTree(tree.ndb.db, 1000, true) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err = sut.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + // LazyLoadVersion - should not auto enable fast storage + version, err := sut.LazyLoadVersion(1) + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + // Load - should not auto enable fast storage + version, err = sut.Load() + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + // LoadVersion - should not auto enable fast storage + version, err = sut.LoadVersion(1) + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + // LoadVersionForOverwriting - should not auto enable fast storage + version, err = sut.LoadVersionForOverwriting(1) + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + t.Run("Mutable tree", func(t *testing.T) { + for _, kv := range mirror { + v, err := sut.Get([]byte(kv[0])) + require.NoError(t, err) + require.Equal(t, []byte(kv[1]), v) + } + }) + + t.Run("Immutable tree", func(t *testing.T) { + immutableTree, err := sut.GetImmutable(sut.version) + require.NoError(t, err) + + for _, kv := range mirror { + v, err := immutableTree.Get([]byte(kv[0])) + require.NoError(t, err) + require.Equal(t, []byte(kv[1]), v) + } + }) +} + +func TestNoFastStorageUpgrade_Integration_SaveVersion_Load_Iterate_Success(t *testing.T) { + // Setup + tree, mirror := setupTreeAndMirror(t, 100, true) + + isFastCacheEnabled, err := tree.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err := tree.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + // Should Not auto enable in save version + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + isFastCacheEnabled, err = tree.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err = tree.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + sut, _ := NewMutableTree(tree.ndb.db, 1000, true) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + isUpgradeable, err = sut.IsUpgradeable() + require.False(t, isUpgradeable) + require.NoError(t, err) + + // Load - should not auto enable fast storage + version, err := sut.Load() + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = sut.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + // Load - should not auto enable fast storage + version, err = sut.Load() + require.NoError(t, err) + require.Equal(t, int64(1), version) + + isFastCacheEnabled, err = tree.IsFastCacheEnabled() + require.NoError(t, err) + require.False(t, isFastCacheEnabled) + + // Test that the mutable tree iterates as expected + t.Run("Mutable tree", func(t *testing.T) { + i := 0 + sut.Iterate(func(k, v []byte) bool { + require.Equal(t, []byte(mirror[i][0]), k) + require.Equal(t, []byte(mirror[i][1]), v) + i++ + return false + }) + }) + + // Test that the immutable tree iterates as expected + t.Run("Immutable tree", func(t *testing.T) { + immutableTree, err := sut.GetImmutable(sut.version) + require.NoError(t, err) + + i := 0 + immutableTree.Iterate(func(k, v []byte) bool { + require.Equal(t, []byte(mirror[i][0]), k) + require.Equal(t, []byte(mirror[i][1]), v) + i++ + return false + }) + }) +} diff --git a/nodedb_test.go b/nodedb_test.go index 389e9d7ff..d889d6b1a 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -279,7 +279,7 @@ func makeHashes(b *testing.B, seed int64) [][]byte { func makeAndPopulateMutableTree(tb testing.TB) *MutableTree { memDB := db.NewMemDB() - tree, err := NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}) + tree, err := NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 9}, false) require.NoError(tb, err) for i := 0; i < 1e4; i++ { diff --git a/proof.go b/proof.go index 51a8860e2..37f9242f8 100644 --- a/proof.go +++ b/proof.go @@ -92,7 +92,7 @@ func (pin ProofInnerNode) Hash(childHash []byte) ([]byte, error) { } } if err != nil { - return nil, fmt.Errorf("Failed to hash ProofInnerNode: %v", err) + return nil, fmt.Errorf("failed to hash ProofInnerNode: %v", err) } _, err = hasher.Write(buf.Bytes()) diff --git a/proof_iavl_test.go b/proof_iavl_test.go index cf3fcd877..7047309cc 100644 --- a/proof_iavl_test.go +++ b/proof_iavl_test.go @@ -12,7 +12,7 @@ import ( ) func TestProofOp(t *testing.T) { - tree, err := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil) + tree, err := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil, false) require.NoError(t, err) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { diff --git a/proof_ics23_test.go b/proof_ics23_test.go index db682a969..fd0a6d196 100644 --- a/proof_ics23_test.go +++ b/proof_ics23_test.go @@ -271,7 +271,7 @@ func GetNonKey(allkeys [][]byte, loc Where) []byte { // BuildTree creates random key/values and stores in tree // returns a list of all keys in sorted order func BuildTree(size int, cacheSize int) (itree *MutableTree, keys [][]byte, err error) { - tree, _ := NewMutableTree(db.NewMemDB(), cacheSize) + tree, _ := NewMutableTree(db.NewMemDB(), cacheSize, false) // insert lots of info and store the bytes keys = make([][]byte, size) diff --git a/proof_range.go b/proof_range.go index 345bbe90d..ada2a3e15 100644 --- a/proof_range.go +++ b/proof_range.go @@ -376,7 +376,7 @@ func RangeProofFromProto(pbProof *iavlproto.RangeProof) (RangeProof, error) { // If keyEnd-1 exists, no later leaves will be included. // If keyStart >= keyEnd and both not nil, errors out. // Limit is never exceeded. -//nolint:unparam + func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { return nil, nil, nil, fmt.Errorf("if keyStart and keyEnd are present, need keyStart < keyEnd") diff --git a/repair_test.go b/repair_test.go index c60923d65..8d17355e0 100644 --- a/repair_test.go +++ b/repair_test.go @@ -32,7 +32,7 @@ func TestRepair013Orphans(t *testing.T) { assert.EqualValues(t, 8, repaired) // Load the database. - tree, err := NewMutableTreeWithOpts(db, 0, &Options{Sync: true}) + tree, err := NewMutableTreeWithOpts(db, 0, &Options{Sync: true}, false) require.NoError(t, err) version, err := tree.Load() require.NoError(t, err) diff --git a/testutils_test.go b/testutils_test.go index d45890d82..85b0f1eeb 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -43,7 +43,7 @@ func b2i(bz []byte) int { // Construct a MutableTree func getTestTree(cacheSize int) (*MutableTree, error) { - return NewMutableTreeWithOpts(db.NewMemDB(), cacheSize, nil) + return NewMutableTreeWithOpts(db.NewMemDB(), cacheSize, nil, false) } // Convenience for a new node @@ -328,7 +328,7 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.StopTimer() - t, err := NewMutableTree(db, 100000) + t, err := NewMutableTree(db, 100000, false) require.NoError(b, err) value := []byte{} diff --git a/tree_random_test.go b/tree_random_test.go index 8816657bf..f9aa3f698 100644 --- a/tree_random_test.go +++ b/tree_random_test.go @@ -82,7 +82,7 @@ func testRandomOperations(t *testing.T, randSeed int64) { if !(r.Float64() < cacheChance) { cacheSize = 0 } - tree, err = NewMutableTreeWithOpts(levelDB, cacheSize, options) + tree, err = NewMutableTreeWithOpts(levelDB, cacheSize, options, false) require.NoError(t, err) version, err = tree.Load() require.NoError(t, err) diff --git a/tree_test.go b/tree_test.go index f5eaa21cf..c02b0de1a 100644 --- a/tree_test.go +++ b/tree_test.go @@ -51,7 +51,7 @@ func TestVersionedRandomTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100, false) require.NoError(err) versions := 50 keysPerVersion := 30 @@ -133,7 +133,7 @@ func TestTreeHash(t *testing.T) { require.Len(t, expectHashes, versions, "must have expected hashes for all versions") r := rand.New(rand.NewSource(randSeed)) - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) keys := make([][]byte, 0, versionOps) @@ -184,7 +184,7 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100, false) require.NoError(err) singleVersionTree, err := getTestTree(0) require.NoError(err) @@ -234,7 +234,7 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100, false) require.NoError(err) singleVersionTree, err := getTestTree(0) require.NoError(err) @@ -329,7 +329,7 @@ func TestVersionedEmptyTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(err) hash, v, err := tree.SaveVersion() @@ -368,7 +368,7 @@ func TestVersionedEmptyTree(t *testing.T) { // Now reload the tree. - tree, err = NewMutableTree(d, 0) + tree, err = NewMutableTree(d, 0, false) require.NoError(err) tree.Load() @@ -387,7 +387,7 @@ func TestVersionedTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(err) // We start with empty database. @@ -438,7 +438,7 @@ func TestVersionedTree(t *testing.T) { // Recreate a new tree and load it, to make sure it works in this // scenario. - tree, err = NewMutableTree(d, 100) + tree, err = NewMutableTree(d, 100, false) require.NoError(err) _, err = tree.Load() require.NoError(err) @@ -492,7 +492,7 @@ func TestVersionedTree(t *testing.T) { require.EqualValues(hash3, hash4) require.NotNil(hash4) - tree, err = NewMutableTree(d, 100) + tree, err = NewMutableTree(d, 100, false) require.NoError(err) _, err = tree.Load() require.NoError(err) @@ -607,7 +607,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(t, err) tree.Set([]byte("key0"), []byte("val0")) @@ -708,7 +708,7 @@ func TestVersionedTreeSpecialCase(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(err) tree.Set([]byte("key1"), []byte("val0")) @@ -733,7 +733,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree, err := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100, false) require.NoError(err) tree.Set([]byte("key1"), []byte("val0")) @@ -747,7 +747,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { tree.Set([]byte("key2"), []byte("val2")) tree.SaveVersion() - tree, err = NewMutableTree(d, 100) + tree, err = NewMutableTree(d, 100, false) require.NoError(err) _, err = tree.Load() require.NoError(err) @@ -793,7 +793,7 @@ func TestVersionedTreeSpecialCase3(t *testing.T) { func TestVersionedTreeSaveAndLoad(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(err) // Loading with an empty root is a no-op. @@ -819,7 +819,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { require.Equal(int64(6), tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. - ntree, err := NewMutableTree(d, 0) + ntree, err := NewMutableTree(d, 0, false) require.NoError(err) ntree.Load() @@ -883,7 +883,7 @@ func TestVersionedCheckpoints(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree, err := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100, false) require.NoError(err) versions := 50 keysPerVersion := 10 @@ -1010,7 +1010,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) tree.Set([]byte("U"), []byte("XamDUtiJ")) @@ -1129,7 +1129,7 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree, err := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(err) versions := 20 keysPerVersion := 100 @@ -1253,7 +1253,7 @@ func TestOrphans(t *testing.T) { // Then randomly delete versions other than the first and last until only those two remain // Any remaining orphan nodes should either have fromVersion == firstVersion || toVersion == lastVersion require := require.New(t) - tree, err := NewMutableTree(db.NewMemDB(), 100) + tree, err := NewMutableTree(db.NewMemDB(), 100, false) require.NoError(err) NUMVERSIONS := 100 @@ -1416,7 +1416,7 @@ func TestOverwrite(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) // Set one kv pair and save version 1 @@ -1430,7 +1430,7 @@ func TestOverwrite(t *testing.T) { require.NoError(err, "SaveVersion should not fail") // Reload tree at version 1 - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) _, err = tree.LoadVersion(int64(1)) require.NoError(err, "LoadVersion should not fail") @@ -1450,7 +1450,7 @@ func TestOverwriteEmpty(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) // Save empty version 1 @@ -1485,7 +1485,7 @@ func TestLoadVersionForOverwriting(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) maxLength := 100 @@ -1497,12 +1497,12 @@ func TestLoadVersionForOverwriting(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) targetVersion, _ := tree.LoadVersionForOverwriting(int64(maxLength * 2)) require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) _, err = tree.LoadVersionForOverwriting(int64(maxLength / 2)) require.NoError(err, "LoadVersion should not fail") @@ -1526,7 +1526,7 @@ func TestLoadVersionForOverwriting(t *testing.T) { require.NoError(err, "SaveVersion should not fail, overwrite was allowed") // Reload tree at version 50, the latest tree version is 52 - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) _, err = tree.LoadVersion(int64(maxLength / 2)) require.NoError(err, "LoadVersion should not fail") @@ -1559,7 +1559,7 @@ func TestDeleteVersionsCompare(t *testing.T) { const fromLength = 5 { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) versions := make([]int64, 0, maxLength) @@ -1573,7 +1573,7 @@ func TestDeleteVersionsCompare(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) targetVersion, err := tree.LoadVersion(int64(maxLength)) require.NoError(err) @@ -1586,7 +1586,7 @@ func TestDeleteVersionsCompare(t *testing.T) { } { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) versions := make([]int64, 0, maxLength) @@ -1600,7 +1600,7 @@ func TestDeleteVersionsCompare(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) targetVersion, err := tree.LoadVersion(int64(maxLength)) require.NoError(err) @@ -1615,7 +1615,7 @@ func TestDeleteVersionsCompare(t *testing.T) { } { mdb := db.NewMemDB() - tree, err := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0, false) require.NoError(err) versions := make([]int64, 0, maxLength) @@ -1629,7 +1629,7 @@ func TestDeleteVersionsCompare(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree, err = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0, false) require.NoError(err) targetVersion, err := tree.LoadVersion(int64(maxLength)) require.NoError(err) @@ -1658,7 +1658,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { defer d.Close() defer os.RemoveAll("./bench.db") - tree, err := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0, false) require.NoError(b, err) for v := 1; v < numVersions; v++ { for i := 0; i < numKeysPerVersion; i++ { @@ -1670,7 +1670,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { b.Run("LoadAndDelete", func(b *testing.B) { for n := 0; n < b.N; n++ { b.StopTimer() - tree, err = NewMutableTree(d, 0) + tree, err = NewMutableTree(d, 0, false) require.NoError(b, err) runtime.GC() b.StartTimer() @@ -1694,7 +1694,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { func TestLoadVersionForOverwritingCase2(t *testing.T) { require := require.New(t) - tree, _ := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil) + tree, _ := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil, false) for i := byte(0); i < 20; i++ { tree.Set([]byte{i}, []byte{i}) @@ -1756,7 +1756,7 @@ func TestLoadVersionForOverwritingCase2(t *testing.T) { func TestLoadVersionForOverwritingCase3(t *testing.T) { require := require.New(t) - tree, err := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil) + tree, err := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil, false) require.NoError(err) for i := byte(0); i < 20; i++ { @@ -1887,7 +1887,7 @@ func Benchmark_GetWithIndex(b *testing.B) { const numKeyVals = 100000 - t, err := NewMutableTree(db, numKeyVals) + t, err := NewMutableTree(db, numKeyVals, false) require.NoError(b, err) keys := make([][]byte, 0, numKeyVals) @@ -1939,7 +1939,7 @@ func Benchmark_GetByIndex(b *testing.B) { const numKeyVals = 100000 - t, err := NewMutableTree(db, numKeyVals) + t, err := NewMutableTree(db, numKeyVals, false) require.NoError(b, err) for i := 0; i < numKeyVals; i++ { @@ -2015,7 +2015,7 @@ func TestNodeCacheStatisic(t *testing.T) { opts := &Options{Stat: stat} db, err := db.NewDB("test", db.MemDBBackend, "") require.NoError(t, err) - mt, err := NewMutableTreeWithOpts(db, tc.cacheSize, opts) + mt, err := NewMutableTreeWithOpts(db, tc.cacheSize, opts, false) require.NoError(t, err) for i := 0; i < numKeyVals; i++ {