From c59cd90939480a8078180dc8251abb28cdaf8b52 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 27 Aug 2021 23:11:27 +0800 Subject: [PATCH 01/36] Clarify db interface godoc --- db/types.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/db/types.go b/db/types.go index bab74bf8c5ed..39dc365925e6 100644 --- a/db/types.go +++ b/db/types.go @@ -107,6 +107,8 @@ type DBReader interface { // DBWriter is a write-only transaction interface. // It is safe for concurrent writes, following an optimistic (OCC) strategy, detecting any write // conflicts and returning an error on commit, rather than locking the DB. +// Callers must call Commit or Discard when done with the transaction. +// // This can be used to wrap a write-optimized batch object if provided by the backend implementation. type DBWriter interface { // Set sets the value for the given key, replacing it if it already exists. @@ -136,29 +138,32 @@ type DBReadWriter interface { // // Callers must make sure the iterator is valid before calling any methods on it, otherwise // these methods will panic. This is in part caused by most backend databases using this convention. +// Note that the iterator is invalid on contruction: Next() must be called to initialize it to its +// starting position. // // As with DBReader, keys and values should be considered read-only, and must be copied before they are // modified. // // Typical usage: // -// var itr Iterator = ... -// defer itr.Close() +// var itr Iterator = ... +// defer itr.Close() // -// for ; itr.Valid(); itr.Next() { -// k, v := itr.Key(); itr.Value() -// ... -// } -// if err := itr.Error(); err != nil { -// ... -// } +// for itr.Next() { +// k, v := itr.Key(); itr.Value() +// ... +// } +// if err := itr.Error(); err != nil { +// ... +// } type Iterator interface { // Domain returns the start (inclusive) and end (exclusive) limits of the iterator. // CONTRACT: start, end readonly []byte Domain() (start []byte, end []byte) // Next moves the iterator to the next key in the database, as defined by order of iteration; - // returns whether the iterator is valid. Once invalid, it remains invalid forever. + // returns whether the iterator is valid. + // Once this function returns false, the iterator remains invalid forever. Next() bool // Key returns the key at the current position. Panics if the iterator is invalid. From c4b45539b24893a146f315433f8d9c1c13cafb61 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 17 Aug 2021 12:44:27 +0800 Subject: [PATCH 02/36] Add prefixed db read/writers --- db/dbtest/util.go | 54 ++++++++++++ db/prefix/prefix.go | 159 +++++++++++++++++++++++++++++++++++ db/prefix/prefix_iterator.go | 114 +++++++++++++++++++++++++ db/prefix/prefix_test.go | 157 ++++++++++++++++++++++++++++++++++ 4 files changed, 484 insertions(+) create mode 100644 db/dbtest/util.go create mode 100644 db/prefix/prefix.go create mode 100644 db/prefix/prefix_iterator.go create mode 100644 db/prefix/prefix_test.go diff --git a/db/dbtest/util.go b/db/dbtest/util.go new file mode 100644 index 000000000000..f65ac87e9fec --- /dev/null +++ b/db/dbtest/util.go @@ -0,0 +1,54 @@ +package dbtest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/cosmos-sdk/db" +) + +func AssertNext(t *testing.T, itr dbm.Iterator, expected bool) { + t.Helper() + require.Equal(t, expected, itr.Next()) +} + +func AssertDomain(t *testing.T, itr dbm.Iterator, start, end []byte) { + t.Helper() + ds, de := itr.Domain() + assert.Equal(t, start, ds, "checkDomain domain start incorrect") + assert.Equal(t, end, de, "checkDomain domain end incorrect") +} + +func AssertItem(t *testing.T, itr dbm.Iterator, key []byte, value []byte) { + t.Helper() + v := itr.Value() + k := itr.Key() + assert.Exactly(t, key, k) + assert.Exactly(t, value, v) +} + +func AssertInvalid(t *testing.T, itr dbm.Iterator) { + t.Helper() + AssertNext(t, itr, false) + AssertKeyPanics(t, itr) + AssertValuePanics(t, itr) +} + +func AssertKeyPanics(t *testing.T, itr dbm.Iterator) { + t.Helper() + assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") +} + +func AssertValue(t *testing.T, db dbm.DBReader, key []byte, valueWanted []byte) { + t.Helper() + valueGot, err := db.Get(key) + assert.NoError(t, err) + assert.Equal(t, valueWanted, valueGot) +} + +func AssertValuePanics(t *testing.T, itr dbm.Iterator) { + t.Helper() + assert.Panics(t, func() { itr.Value() }) +} diff --git a/db/prefix/prefix.go b/db/prefix/prefix.go new file mode 100644 index 000000000000..26ac2741d778 --- /dev/null +++ b/db/prefix/prefix.go @@ -0,0 +1,159 @@ +package prefix + +import ( + dbm "github.com/cosmos/cosmos-sdk/db" +) + +// Prefix Reader/Writer lets you namespace multiple DBs within a single DB. +type prefixR struct { + db dbm.DBReader + prefix []byte +} + +type prefixRW struct { + db dbm.DBReadWriter + prefix []byte +} + +var _ dbm.DBReader = (*prefixR)(nil) +var _ dbm.DBReadWriter = (*prefixRW)(nil) + +func NewPrefixReader(db dbm.DBReader, prefix []byte) prefixR { + return prefixR{ + prefix: prefix, + db: db, + } +} + +func NewPrefixReadWriter(db dbm.DBReadWriter, prefix []byte) prefixRW { + return prefixRW{ + prefix: prefix, + db: db, + } +} + +func prefixed(prefix []byte, key []byte) []byte { + return append(prefix, key...) +} + +// Get implements DBReader. +func (pdb prefixR) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, dbm.ErrKeyEmpty + } + return pdb.db.Get(prefixed(pdb.prefix, key)) +} + +// Has implements DBReader. +func (pdb prefixR) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, dbm.ErrKeyEmpty + } + return pdb.db.Has(prefixed(pdb.prefix, key)) +} + +// Iterator implements DBReader. +func (pdb prefixR) Iterator(start, end []byte) (dbm.Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, dbm.ErrKeyEmpty + } + + var pend []byte + if end == nil { + pend = cpIncr(pdb.prefix) + } else { + pend = prefixed(pdb.prefix, end) + } + itr, err := pdb.db.Iterator(prefixed(pdb.prefix, start), pend) + if err != nil { + return nil, err + } + return newPrefixIterator(pdb.prefix, start, end, itr), nil +} + +// ReverseIterator implements DBReader. +func (pdb prefixR) ReverseIterator(start, end []byte) (dbm.Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, dbm.ErrKeyEmpty + } + + var pend []byte + if end == nil { + pend = cpIncr(pdb.prefix) + } else { + pend = prefixed(pdb.prefix, end) + } + ritr, err := pdb.db.ReverseIterator(prefixed(pdb.prefix, start), pend) + if err != nil { + return nil, err + } + return newPrefixIterator(pdb.prefix, start, end, ritr), nil +} + +// Discard implements DBReader. +func (pdb prefixR) Discard() error { return pdb.db.Discard() } + +// Set implements DBReadWriter. +func (pdb prefixRW) Set(key []byte, value []byte) error { + if len(key) == 0 { + return dbm.ErrKeyEmpty + } + return pdb.db.Set(prefixed(pdb.prefix, key), value) +} + +// Delete implements DBReadWriter. +func (pdb prefixRW) Delete(key []byte) error { + if len(key) == 0 { + return dbm.ErrKeyEmpty + } + return pdb.db.Delete(prefixed(pdb.prefix, key)) +} + +// Get implements DBReadWriter. +func (pdb prefixRW) Get(key []byte) ([]byte, error) { + return NewPrefixReader(pdb.db, pdb.prefix).Get(key) +} + +// Has implements DBReadWriter. +func (pdb prefixRW) Has(key []byte) (bool, error) { + return NewPrefixReader(pdb.db, pdb.prefix).Has(key) +} + +// Iterator implements DBReadWriter. +func (pdb prefixRW) Iterator(start, end []byte) (dbm.Iterator, error) { + return NewPrefixReader(pdb.db, pdb.prefix).Iterator(start, end) +} + +// ReverseIterator implements DBReadWriter. +func (pdb prefixRW) ReverseIterator(start, end []byte) (dbm.Iterator, error) { + return NewPrefixReader(pdb.db, pdb.prefix).ReverseIterator(start, end) +} + +// Close implements DBReadWriter. +func (pdb prefixRW) Commit() error { return pdb.db.Commit() } + +// Discard implements DBReadWriter. +func (pdb prefixRW) Discard() error { return pdb.db.Discard() } + +// Returns a slice of the same length (big endian), but incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) +// CONTRACT: len(bz) > 0 +func cpIncr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } + ret = make([]byte, len(bz)) + copy(ret, bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] < byte(0xFF) { + ret[i]++ + return + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } + } + return nil +} diff --git a/db/prefix/prefix_iterator.go b/db/prefix/prefix_iterator.go new file mode 100644 index 000000000000..7a8169e11986 --- /dev/null +++ b/db/prefix/prefix_iterator.go @@ -0,0 +1,114 @@ +package prefix + +import ( + "bytes" + "fmt" + + dbm "github.com/cosmos/cosmos-sdk/db" +) + +// IteratePrefix is a convenience function for iterating over a key domain +// restricted by prefix. +func IteratePrefix(db dbm.DBReader, prefix []byte) (dbm.Iterator, error) { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = prefix + end = cpIncr(prefix) + } + itr, err := db.Iterator(start, end) + if err != nil { + return nil, err + } + return itr, nil +} + +// Strips prefix while iterating from Iterator. +type prefixDBIterator struct { + prefix []byte + start []byte + end []byte + source dbm.Iterator + err error +} + +var _ dbm.Iterator = (*prefixDBIterator)(nil) + +func newPrefixIterator(prefix, start, end []byte, source dbm.Iterator) *prefixDBIterator { + return &prefixDBIterator{ + prefix: prefix, + start: start, + end: end, + source: source, + } +} + +// Domain implements Iterator. +func (itr *prefixDBIterator) Domain() (start []byte, end []byte) { + return itr.start, itr.end +} + +func (itr *prefixDBIterator) valid() bool { + if itr.err != nil { + return false + } + + key := itr.source.Key() + if len(key) < len(itr.prefix) || !bytes.Equal(key[:len(itr.prefix)], itr.prefix) { + itr.err = fmt.Errorf("received invalid key from backend: %x (expected prefix %x)", + key, itr.prefix) + return false + } + + return true +} + +// Next implements Iterator. +func (itr *prefixDBIterator) Next() bool { + if !itr.source.Next() { + return false + } + if !bytes.HasPrefix(itr.source.Key(), itr.prefix) { + return false + } + // Empty keys are not allowed, so if a key exists in the database that exactly matches the + // prefix we need to skip it. + if bytes.Equal(itr.source.Key(), itr.prefix) { + return itr.Next() + } + return true +} + +// Next implements Iterator. +func (itr *prefixDBIterator) Key() []byte { + itr.assertIsValid() + key := itr.source.Key() + return key[len(itr.prefix):] // we have checked the key in Valid() +} + +// Value implements Iterator. +func (itr *prefixDBIterator) Value() []byte { + itr.assertIsValid() + return itr.source.Value() +} + +// Error implements Iterator. +func (itr *prefixDBIterator) Error() error { + if err := itr.source.Error(); err != nil { + return err + } + return itr.err +} + +// Close implements Iterator. +func (itr *prefixDBIterator) Close() error { + return itr.source.Close() +} + +func (itr *prefixDBIterator) assertIsValid() { + if !itr.valid() { + panic("iterator is invalid") + } +} diff --git a/db/prefix/prefix_test.go b/db/prefix/prefix_test.go new file mode 100644 index 000000000000..2a09eaf4e604 --- /dev/null +++ b/db/prefix/prefix_test.go @@ -0,0 +1,157 @@ +package prefix_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/cosmos-sdk/db" + "github.com/cosmos/cosmos-sdk/db/dbtest" + "github.com/cosmos/cosmos-sdk/db/memdb" + pfx "github.com/cosmos/cosmos-sdk/db/prefix" +) + +func fillDBWithStuff(t *testing.T, dbw dbm.DBWriter) { + // Under "key" prefix + require.NoError(t, dbw.Set([]byte("key"), []byte("value"))) + require.NoError(t, dbw.Set([]byte("key1"), []byte("value1"))) + require.NoError(t, dbw.Set([]byte("key2"), []byte("value2"))) + require.NoError(t, dbw.Set([]byte("key3"), []byte("value3"))) + require.NoError(t, dbw.Set([]byte("something"), []byte("else"))) + require.NoError(t, dbw.Set([]byte("k"), []byte("val"))) + require.NoError(t, dbw.Set([]byte("ke"), []byte("valu"))) + require.NoError(t, dbw.Set([]byte("kee"), []byte("valuu"))) + require.NoError(t, dbw.Commit()) +} + +func mockDBWithStuff(t *testing.T) dbm.DBConnection { + db := memdb.NewDB() + fillDBWithStuff(t, db.Writer()) + return db +} + +func makePrefixReader(t *testing.T, db dbm.DBConnection, pre []byte) dbm.DBReader { + view := db.Reader() + require.NotNil(t, view) + return pfx.NewPrefixReader(view, pre) +} + +func TestPrefixDBSimple(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + dbtest.AssertValue(t, pdb, []byte("key"), nil) + dbtest.AssertValue(t, pdb, []byte("key1"), nil) + dbtest.AssertValue(t, pdb, []byte("1"), []byte("value1")) + dbtest.AssertValue(t, pdb, []byte("key2"), nil) + dbtest.AssertValue(t, pdb, []byte("2"), []byte("value2")) + dbtest.AssertValue(t, pdb, []byte("key3"), nil) + dbtest.AssertValue(t, pdb, []byte("3"), []byte("value3")) + dbtest.AssertValue(t, pdb, []byte("something"), nil) + dbtest.AssertValue(t, pdb, []byte("k"), nil) + dbtest.AssertValue(t, pdb, []byte("ke"), nil) + dbtest.AssertValue(t, pdb, []byte("kee"), nil) +} + +func TestPrefixDBIterator1(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + itr, err := pdb.Iterator(nil, nil) + require.NoError(t, err) + dbtest.AssertDomain(t, itr, nil, nil) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("1"), []byte("value1")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("2"), []byte("value2")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("3"), []byte("value3")) + dbtest.AssertNext(t, itr, false) + dbtest.AssertInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator1(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + itr, err := pdb.ReverseIterator(nil, nil) + require.NoError(t, err) + dbtest.AssertDomain(t, itr, nil, nil) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("3"), []byte("value3")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("2"), []byte("value2")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("1"), []byte("value1")) + dbtest.AssertNext(t, itr, false) + dbtest.AssertInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator5(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + itr, err := pdb.ReverseIterator([]byte("1"), nil) + require.NoError(t, err) + dbtest.AssertDomain(t, itr, []byte("1"), nil) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("3"), []byte("value3")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("2"), []byte("value2")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("1"), []byte("value1")) + dbtest.AssertNext(t, itr, false) + dbtest.AssertInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator6(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + itr, err := pdb.ReverseIterator([]byte("2"), nil) + require.NoError(t, err) + dbtest.AssertDomain(t, itr, []byte("2"), nil) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("3"), []byte("value3")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("2"), []byte("value2")) + dbtest.AssertNext(t, itr, false) + dbtest.AssertInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator7(t *testing.T) { + pdb := makePrefixReader(t, mockDBWithStuff(t), []byte("key")) + + itr, err := pdb.ReverseIterator(nil, []byte("2")) + require.NoError(t, err) + dbtest.AssertDomain(t, itr, nil, []byte("2")) + dbtest.AssertNext(t, itr, true) + dbtest.AssertItem(t, itr, []byte("1"), []byte("value1")) + dbtest.AssertNext(t, itr, false) + dbtest.AssertInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBViewVersion(t *testing.T) { + prefix := []byte("key") + db := memdb.NewDB() + fillDBWithStuff(t, db.Writer()) + id, err := db.SaveNextVersion() + require.NoError(t, err) + pdb := pfx.NewPrefixReadWriter(db.ReadWriter(), prefix) + + pdb.Set([]byte("1"), []byte("newvalue1")) + pdb.Delete([]byte("2")) + pdb.Set([]byte("4"), []byte("newvalue4")) + pdb.Discard() + + dbview, err := db.ReaderAt(id) + require.NotNil(t, dbview) + require.NoError(t, err) + view := pfx.NewPrefixReader(dbview, prefix) + require.NotNil(t, view) + defer view.Discard() + + dbtest.AssertValue(t, view, []byte("1"), []byte("value1")) + dbtest.AssertValue(t, view, []byte("2"), []byte("value2")) + dbtest.AssertValue(t, view, []byte("4"), nil) +} From 94c2c175b51d216b37ff4f57e694f8cc169c002d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 12 Aug 2021 13:08:14 +0800 Subject: [PATCH 03/36] Add db adapter --- db/adapter.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 db/adapter.go diff --git a/db/adapter.go b/db/adapter.go new file mode 100644 index 000000000000..66aeee7e309e --- /dev/null +++ b/db/adapter.go @@ -0,0 +1,23 @@ +package db + +type readerRWAdapter struct{ DBReader } + +// ReaderAsReadWriter returns a ReadWriter that forwards to a reader and errors if writes are +// attempted. Can be used to pass a Reader when a ReadWriter is expected +// but no writes will actually occur. +func ReaderAsReadWriter(r DBReader) DBReadWriter { + return readerRWAdapter{r} +} + +func (readerRWAdapter) Set([]byte, []byte) error { + return ErrReadOnly +} + +func (readerRWAdapter) Delete([]byte) error { + return ErrReadOnly +} + +func (rw readerRWAdapter) Commit() error { + rw.Discard() + return nil +} From c164920c7c8fbd127c0c5589be10d1ecd56942bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Zdyba=C5=82?= Date: Fri, 14 May 2021 19:36:42 +0200 Subject: [PATCH 04/36] Add SMT store type (#8507) * Initial SMT store type SMT (Sparse Merkle Tree) is intended to replace IAVL. New type implements same interfaces as iavl.Store. * Add iteration support to SMT Sparse Merkle Tree does not support iteration over keys in order. To provide drop-in replacement for IAVL, Iterator and ReverseIterator has to be implemented. SMT Store implementation use the underlying KV store to: - maintain a list of keys (under a prefix) - iterate over a keys Values are stored only in SMT. * Migrate to smt v0.1.1 * Extra test for SMT iterator * CommitStore implementation for SMT store * Use interface instead of concrete type * Add telemetry to SMT store * SMT: version->root mapping, cleanup * SMT proofs - initial code * Tests for SMT store ProofOp implementation * Fix linter errors * Use simple 1 byte KV-store prefixes * Improve assertions in tests * Use mutex properly * Store data in ADR-040-compatible way SMT stores: * key -> hash(key, value) KV store stores: * key->value in "bucket 1", * hash(key, value) -> key in "bucket 2". --- go.mod | 1 + go.sum | 2 + store/rootmulti/proof.go | 7 ++ store/rootmulti/store.go | 2 +- store/smt/iterator.go | 101 ++++++++++++++++++ store/smt/iterator_test.go | 113 ++++++++++++++++++++ store/smt/proof.go | 92 ++++++++++++++++ store/smt/proof_test.go | 68 ++++++++++++ store/smt/store.go | 211 +++++++++++++++++++++++++++++++++++++ store/smt/store_test.go | 43 ++++++++ store/types/store.go | 4 + 11 files changed, 643 insertions(+), 1 deletion(-) create mode 100644 store/smt/iterator.go create mode 100644 store/smt/iterator_test.go create mode 100644 store/smt/proof.go create mode 100644 store/smt/proof_test.go create mode 100644 store/smt/store.go create mode 100644 store/smt/store_test.go diff --git a/go.mod b/go.mod index c08cb4d60bd1..8cb58692c56f 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/improbable-eng/grpc-web v0.14.1 github.com/jhump/protoreflect v1.10.1 github.com/kr/text v0.2.0 // indirect + github.com/lazyledger/smt v0.2.0 // indirect github.com/magiconair/properties v1.8.5 github.com/mattn/go-isatty v0.0.14 github.com/onsi/ginkgo v1.16.4 // indirect diff --git a/go.sum b/go.sum index 7aeaff0205a4..925c4aca6b6e 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lazyledger/smt v0.1.1 h1:EiZZnov3ixjvqBYvlPqBqkunarm0wU0tJhWdJxCgbpA= +github.com/lazyledger/smt v0.1.1/go.mod h1:9+Pb2/tg1PvEgW7aFx4bFhDE4bvbI03zuJ8kb7nJ9Jc= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= diff --git a/store/rootmulti/proof.go b/store/rootmulti/proof.go index fc8925b7f20d..247e3867b80d 100644 --- a/store/rootmulti/proof.go +++ b/store/rootmulti/proof.go @@ -3,6 +3,7 @@ package rootmulti import ( "github.com/tendermint/tendermint/crypto/merkle" + "github.com/cosmos/cosmos-sdk/store/smt" storetypes "github.com/cosmos/cosmos-sdk/store/types" ) @@ -25,3 +26,9 @@ func DefaultProofRuntime() (prt *merkle.ProofRuntime) { prt.RegisterOpDecoder(storetypes.ProofOpSimpleMerkleCommitment, storetypes.CommitmentOpDecoder) return } + +func SMTProofRuntime() (prt *merkle.ProofRuntime) { + prt = merkle.NewProofRuntime() + prt.RegisterOpDecoder(smt.ProofType, smt.ProofDecoder) + return prt +} diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index d00302e7aa6f..a8b4b770c032 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -620,7 +620,7 @@ func (rs *Store) SetInitialVersion(version int64) error { // If the store is wrapped with an inter-block cache, we must first unwrap // it to get the underlying IAVL store. store = rs.GetCommitKVStore(key) - store.(*iavl.Store).SetInitialVersion(version) + store.(types.StoreWithInitialVersion).SetInitialVersion(version) } } diff --git a/store/smt/iterator.go b/store/smt/iterator.go new file mode 100644 index 000000000000..459460d77532 --- /dev/null +++ b/store/smt/iterator.go @@ -0,0 +1,101 @@ +package smt + +import ( + "bytes" + + dbm "github.com/tendermint/tm-db" +) + +type Iterator struct { + store *Store + iter dbm.Iterator +} + +func indexKey(key []byte) []byte { + return append(indexPrefix, key...) +} + +func plainKey(key []byte) []byte { + return key[prefixLen:] +} + +func startKey(key []byte) []byte { + if key == nil { + return dataPrefix + } + return dataKey(key) +} + +func endKey(key []byte) []byte { + if key == nil { + return indexPrefix + } + return dataKey(key) +} + +func newIterator(s *Store, start, end []byte, reverse bool) (*Iterator, error) { + start = startKey(start) + end = endKey(end) + var i dbm.Iterator + var err error + if reverse { + i, err = s.db.ReverseIterator(start, end) + } else { + i, err = s.db.Iterator(start, end) + } + if err != nil { + return nil, err + } + return &Iterator{store: s, iter: i}, nil +} + +// Domain returns the start (inclusive) and end (exclusive) limits of the iterator. +// CONTRACT: start, end readonly []byte +func (i *Iterator) Domain() (start []byte, end []byte) { + start, end = i.iter.Domain() + if bytes.Equal(start, dataPrefix) { + start = nil + } else { + start = plainKey(start) + } + if bytes.Equal(end, indexPrefix) { + end = nil + } else { + end = plainKey(end) + } + return start, end +} + +// Valid returns whether the current iterator is valid. Once invalid, the Iterator remains +// invalid forever. +func (i *Iterator) Valid() bool { + return i.iter.Valid() +} + +// Next moves the iterator to the next key in the database, as defined by order of iteration. +// If Valid returns false, this method will panic. +func (i *Iterator) Next() { + i.iter.Next() +} + +// Key returns the key at the current position. Panics if the iterator is invalid. +// CONTRACT: key readonly []byte +func (i *Iterator) Key() (key []byte) { + return plainKey(i.iter.Key()) +} + +// Value returns the value at the current position. Panics if the iterator is invalid. +// CONTRACT: value readonly []byte +func (i *Iterator) Value() (value []byte) { + return i.store.Get(i.Key()) +} + +// Error returns the last error encountered by the iterator, if any. +func (i *Iterator) Error() error { + return i.iter.Error() +} + +// Close closes the iterator, relasing any allocated resources. +func (i *Iterator) Close() error { + return i.iter.Close() +} diff --git a/store/smt/iterator_test.go b/store/smt/iterator_test.go new file mode 100644 index 000000000000..6a724e665a1e --- /dev/null +++ b/store/smt/iterator_test.go @@ -0,0 +1,113 @@ +package smt_test + +import ( + "bytes" + "sort" + "testing" + + "github.com/cosmos/cosmos-sdk/store/smt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +func TestIteration(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + pairs := []struct{ key, val []byte }{ + {[]byte("foo"), []byte("bar")}, + {[]byte("lorem"), []byte("ipsum")}, + {[]byte("alpha"), []byte("beta")}, + {[]byte("gamma"), []byte("delta")}, + {[]byte("epsilon"), []byte("zeta")}, + {[]byte("eta"), []byte("theta")}, + {[]byte("iota"), []byte("kappa")}, + } + + s := smt.NewStore(dbm.NewMemDB()) + + for _, p := range pairs { + s.Set(p.key, p.val) + } + + // sort test data by key, to get "expected" ordering + sort.Slice(pairs, func(i, j int) bool { + return bytes.Compare(pairs[i].key, pairs[j].key) < 0 + }) + + iter := s.Iterator([]byte("alpha"), []byte("omega")) + for _, p := range pairs { + require.True(iter.Valid()) + require.Equal(p.key, iter.Key()) + require.Equal(p.val, iter.Value()) + iter.Next() + } + assert.False(iter.Valid()) + assert.NoError(iter.Error()) + assert.NoError(iter.Close()) + + iter = s.Iterator(nil, nil) + for _, p := range pairs { + require.True(iter.Valid()) + require.Equal(p.key, iter.Key()) + require.Equal(p.val, iter.Value()) + iter.Next() + } + assert.False(iter.Valid()) + assert.NoError(iter.Error()) + assert.NoError(iter.Close()) + + iter = s.Iterator([]byte("epsilon"), []byte("gamma")) + for _, p := range pairs[1:4] { + require.True(iter.Valid()) + require.Equal(p.key, iter.Key()) + require.Equal(p.val, iter.Value()) + iter.Next() + } + assert.False(iter.Valid()) + assert.NoError(iter.Error()) + assert.NoError(iter.Close()) + + rIter := s.ReverseIterator(nil, nil) + for i := len(pairs) - 1; i >= 0; i-- { + require.True(rIter.Valid()) + require.Equal(pairs[i].key, rIter.Key()) + require.Equal(pairs[i].val, rIter.Value()) + rIter.Next() + } + assert.False(rIter.Valid()) + assert.NoError(rIter.Error()) + assert.NoError(rIter.Close()) + + // delete something, and ensure that iteration still works + s.Delete([]byte("eta")) + + iter = s.Iterator(nil, nil) + for _, p := range pairs { + if !bytes.Equal([]byte("eta"), p.key) { + require.True(iter.Valid()) + require.Equal(p.key, iter.Key()) + require.Equal(p.val, iter.Value()) + iter.Next() + } + } + assert.False(iter.Valid()) + assert.NoError(iter.Error()) + assert.NoError(iter.Close()) +} + +func TestDomain(t *testing.T) { + assert := assert.New(t) + s := smt.NewStore(dbm.NewMemDB()) + + iter := s.Iterator(nil, nil) + start, end := iter.Domain() + assert.Nil(start) + assert.Nil(end) + + iter = s.Iterator([]byte("foo"), []byte("bar")) + start, end = iter.Domain() + assert.Equal([]byte("foo"), start) + assert.Equal([]byte("bar"), end) +} diff --git a/store/smt/proof.go b/store/smt/proof.go new file mode 100644 index 000000000000..6663c27cec82 --- /dev/null +++ b/store/smt/proof.go @@ -0,0 +1,92 @@ +package smt + +import ( + "bytes" + "crypto/sha256" + "encoding/gob" + "hash" + + "github.com/cosmos/cosmos-sdk/store/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/lazyledger/smt" + "github.com/tendermint/tendermint/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto" +) + +type HasherType byte + +const ( + SHA256 HasherType = iota +) + +const ( + ProofType = "smt" +) + +type ProofOp struct { + Root []byte + Key []byte + Hasher HasherType + Proof smt.SparseMerkleProof +} + +var _ merkle.ProofOperator = &ProofOp{} + +func NewProofOp(root, key []byte, hasher HasherType, proof smt.SparseMerkleProof) *ProofOp { + return &ProofOp{ + Root: root, + Key: key, + Hasher: hasher, + Proof: proof, + } +} + +func (p *ProofOp) Run(args [][]byte) ([][]byte, error) { + switch len(args) { + case 0: // non-membership proof + if !smt.VerifyProof(p.Proof, p.Root, p.Key, []byte{}, getHasher(p.Hasher)) { + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "proof did not verify absence of key: %s", p.Key) + } + case 1: // membership proof + if !smt.VerifyProof(p.Proof, p.Root, p.Key, args[0], getHasher(p.Hasher)) { + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "proof did not verify existence of key %s with given value %x", p.Key, args[0]) + } + default: + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "args must be length 0 or 1, got: %d", len(args)) + } + return [][]byte{p.Root}, nil +} + +func (p *ProofOp) GetKey() []byte { + return p.Key +} + +func (p *ProofOp) ProofOp() tmmerkle.ProofOp { + var data bytes.Buffer + enc := gob.NewEncoder(&data) + enc.Encode(p) + return tmmerkle.ProofOp{ + Type: "smt", + Key: p.Key, + Data: data.Bytes(), + } +} + +func ProofDecoder(pop tmmerkle.ProofOp) (merkle.ProofOperator, error) { + dec := gob.NewDecoder(bytes.NewBuffer(pop.Data)) + var proof ProofOp + err := dec.Decode(&proof) + if err != nil { + return nil, err + } + return &proof, nil +} + +func getHasher(hasher HasherType) hash.Hash { + switch hasher { + case SHA256: + return sha256.New() + default: + return nil + } +} diff --git a/store/smt/proof_test.go b/store/smt/proof_test.go new file mode 100644 index 000000000000..75e7974bc214 --- /dev/null +++ b/store/smt/proof_test.go @@ -0,0 +1,68 @@ +package smt_test + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + smtstore "github.com/cosmos/cosmos-sdk/store/smt" + "github.com/lazyledger/smt" + dbm "github.com/tendermint/tm-db" +) + +func TestProofOpInterface(t *testing.T) { + hasher := sha256.New() + tree := smt.NewSparseMerkleTree(dbm.NewMemDB(), hasher) + key := []byte("foo") + value := []byte("bar") + root, err := tree.Update(key, value) + require.NoError(t, err) + require.NotEmpty(t, root) + + proof, err := tree.Prove(key) + require.True(t, smt.VerifyProof(proof, root, key, value, hasher)) + + storeProofOp := smtstore.NewProofOp(root, key, smtstore.SHA256, proof) + require.NotNil(t, storeProofOp) + // inclusion proof + r, err := storeProofOp.Run([][]byte{value}) + assert.NoError(t, err) + assert.NotEmpty(t, r) + assert.Equal(t, root, r[0]) + + // inclusion proof - wrong value - should fail + r, err = storeProofOp.Run([][]byte{key}) + assert.Error(t, err) + assert.Empty(t, r) + + // exclusion proof - should fail + r, err = storeProofOp.Run([][]byte{}) + assert.Error(t, err) + assert.Empty(t, r) + + // invalid request - should fail + r, err = storeProofOp.Run([][]byte{key, key}) + assert.Error(t, err) + assert.Empty(t, r) + + // encode + tmProofOp := storeProofOp.ProofOp() + assert.NotNil(t, tmProofOp) + assert.Equal(t, smtstore.ProofType, tmProofOp.Type) + assert.Equal(t, key, tmProofOp.Key, key) + assert.NotEmpty(t, tmProofOp.Data) + + //decode + decoded, err := smtstore.ProofDecoder(tmProofOp) + assert.NoError(t, err) + assert.NotNil(t, decoded) + assert.Equal(t, key, decoded.GetKey()) + + // run proof after decoding + r, err = decoded.Run([][]byte{value}) + assert.NoError(t, err) + assert.NotEmpty(t, r) + assert.Equal(t, root, r[0]) +} diff --git a/store/smt/store.go b/store/smt/store.go new file mode 100644 index 000000000000..b70b097515d9 --- /dev/null +++ b/store/smt/store.go @@ -0,0 +1,211 @@ +package smt + +import ( + "crypto/sha256" + "encoding/binary" + "io" + "sync" + "time" + + "github.com/cosmos/cosmos-sdk/store/cachekv" + "github.com/cosmos/cosmos-sdk/store/tracekv" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" + abci "github.com/tendermint/tendermint/abci/types" + dbm "github.com/tendermint/tm-db" + + "github.com/lazyledger/smt" +) + +var ( + _ types.KVStore = (*Store)(nil) + _ types.CommitStore = (*Store)(nil) + _ types.CommitKVStore = (*Store)(nil) + _ types.Queryable = (*Store)(nil) + _ types.StoreWithInitialVersion = (*Store)(nil) +) + +var ( + prefixLen = 1 + versionsPrefix = []byte{0} + dataPrefix = []byte{1} + indexPrefix = []byte{2} + afterIndex = []byte{3} +) + +// Store Implements types.KVStore and CommitKVStore. +type Store struct { + tree *smt.SparseMerkleTree + db dbm.DB + + version int64 + + opts struct { + initialVersion int64 + pruningOptions types.PruningOptions + } + + mtx sync.RWMutex +} + +func NewStore(underlyingDB dbm.DB) *Store { + return &Store{ + tree: smt.NewSparseMerkleTree(underlyingDB, sha256.New()), + db: underlyingDB, + } +} + +// KVStore interface below: + +func (s *Store) GetStoreType() types.StoreType { + return types.StoreTypeSMT +} + +// CacheWrap branches a store. +func (s *Store) CacheWrap() types.CacheWrap { + return cachekv.NewStore(s) +} + +// CacheWrapWithTrace branches a store with tracing enabled. +func (s *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.CacheWrap { + return cachekv.NewStore(tracekv.NewStore(s, w, tc)) +} + +// Get returns nil iff key doesn't exist. Panics on nil key. +func (s *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "get") + val, err := s.db.Get(dataKey(key)) + if err != nil { + panic(err) + } + return val +} + +// Has checks if a key exists. Panics on nil key. +func (s *Store) Has(key []byte) bool { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "has") + has, err := s.db.Has(dataKey(key)) + return err == nil && has +} + +// Set sets the key. Panics on nil key or value. +func (s *Store) Set(key []byte, value []byte) { + kvHash := sha256.Sum256(append(key, value...)) + + s.mtx.Lock() + defer s.mtx.Unlock() + + err := s.db.Set(dataKey(key), value) + if err != nil { + panic(err.Error()) + } + err = s.db.Set(indexKey(kvHash[:]), key) + if err != nil { + panic(err.Error()) + } + _, err = s.tree.Update(key, kvHash[:]) + if err != nil { + panic(err.Error()) + } +} + +// Delete deletes the key. Panics on nil key. +func (s *Store) Delete(key []byte) { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "delete") + + s.mtx.Lock() + defer s.mtx.Unlock() + + _, _ = s.tree.Delete(key) + + dKey := dataKey(key) + defer func() { + _ = s.db.Delete(dKey) + }() + + value, err := s.db.Get(dKey) + if err != nil { + panic(err.Error()) + } + kvHash := sha256.Sum256(append(key, value...)) + _ = s.db.Delete(indexKey(kvHash[:])) +} + +// Iterator over a domain of keys in ascending order. End is exclusive. +// Start must be less than end, or the Iterator is invalid. +// Iterator must be closed by caller. +// To iterate over entire domain, use store.Iterator(nil, nil) +// CONTRACT: No writes may happen within a domain while an iterator exists over it. +// Exceptionally allowed for cachekv.Store, safe to write in the modules. +func (s *Store) Iterator(start []byte, end []byte) types.Iterator { + iter, err := newIterator(s, start, end, false) + if err != nil { + panic(err.Error()) + } + return iter +} + +// Iterator over a domain of keys in descending order. End is exclusive. +// Start must be less than end, or the Iterator is invalid. +// Iterator must be closed by caller. +// CONTRACT: No writes may happen within a domain while an iterator exists over it. +// Exceptionally allowed for cachekv.Store, safe to write in the modules. +func (s *Store) ReverseIterator(start []byte, end []byte) types.Iterator { + iter, err := newIterator(s, start, end, true) + if err != nil { + panic(err.Error()) + } + return iter +} + +// CommitStore interface below: + +func (s *Store) Commit() types.CommitID { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "commit") + version := s.version + 1 + + if version == 1 && s.opts.initialVersion != 0 { + version = s.opts.initialVersion + } + + s.version = version + + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, uint64(version)) + s.db.Set(append(versionsPrefix, b...), s.tree.Root()) + + return s.LastCommitID() +} + +func (s *Store) LastCommitID() types.CommitID { + return types.CommitID{ + Version: s.version, + Hash: s.tree.Root(), + } +} + +func (s *Store) SetPruning(p types.PruningOptions) { + s.opts.pruningOptions = p +} + +func (s *Store) GetPruning() types.PruningOptions { + return s.opts.pruningOptions +} + +// Queryable interface below: + +func (s *Store) Query(_ abci.RequestQuery) abci.ResponseQuery { + panic("not implemented") +} + +// StoreWithInitialVersion interface below: + +// SetInitialVersion sets the initial version of the SMT tree. It is used when +// starting a new chain at an arbitrary height. +func (s *Store) SetInitialVersion(version int64) { + s.opts.initialVersion = version +} + +func dataKey(key []byte) []byte { + return append(dataPrefix, key...) +} diff --git a/store/smt/store_test.go b/store/smt/store_test.go new file mode 100644 index 000000000000..a655e4b6d320 --- /dev/null +++ b/store/smt/store_test.go @@ -0,0 +1,43 @@ +package smt_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cosmos/cosmos-sdk/store/smt" + dbm "github.com/tendermint/tm-db" +) + +func TestVersioning(t *testing.T) { + s := smt.NewStore(dbm.NewMemDB()) + expectedVersion := int64(0) + + s.Set([]byte("foo"), []byte("bar")) + cid1 := s.Commit() + expectedVersion++ + + assert.Equal(t, expectedVersion, cid1.Version) + assert.NotEmpty(t, cid1.Hash) + + s.Set([]byte("foobar"), []byte("baz")) + cid2 := s.Commit() + expectedVersion++ + + assert.Equal(t, expectedVersion, cid2.Version) + assert.NotEmpty(t, cid2.Hash) + assert.NotEqual(t, cid1.Hash, cid2.Hash) +} + +func TestInitialVersion(t *testing.T) { + s := smt.NewStore(dbm.NewMemDB()) + expectedVersion := int64(42) + + s.SetInitialVersion(expectedVersion) + + s.Set([]byte("foo"), []byte("foobar")) + cid := s.Commit() + + assert.Equal(t, expectedVersion, cid.Version) + assert.NotEmpty(t, cid.Hash) +} diff --git a/store/types/store.go b/store/types/store.go index 2c352c0105cf..4e18c43a3b31 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -289,6 +289,7 @@ const ( StoreTypeIAVL StoreTypeTransient StoreTypeMemory + StoreTypeSMT ) func (st StoreType) String() string { @@ -307,6 +308,9 @@ func (st StoreType) String() string { case StoreTypeMemory: return "StoreTypeMemory" + + case StoreTypeSMT: + return "StoreTypeSMT" } return "unknown store type" From 51bc84b077b2b326ea1b672693e73853af722cfa Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 8 Jul 2021 17:18:39 +0800 Subject: [PATCH 05/36] Add godoc comments --- store/rootmulti/proof.go | 1 + store/smt/proof.go | 1 + 2 files changed, 2 insertions(+) diff --git a/store/rootmulti/proof.go b/store/rootmulti/proof.go index 247e3867b80d..6766779b2dd5 100644 --- a/store/rootmulti/proof.go +++ b/store/rootmulti/proof.go @@ -27,6 +27,7 @@ func DefaultProofRuntime() (prt *merkle.ProofRuntime) { return } +// SMTProofRuntime returns a ProofRuntime for sparse merkle trees. func SMTProofRuntime() (prt *merkle.ProofRuntime) { prt = merkle.NewProofRuntime() prt.RegisterOpDecoder(smt.ProofType, smt.ProofDecoder) diff --git a/store/smt/proof.go b/store/smt/proof.go index 6663c27cec82..e606b0f649b3 100644 --- a/store/smt/proof.go +++ b/store/smt/proof.go @@ -32,6 +32,7 @@ type ProofOp struct { var _ merkle.ProofOperator = &ProofOp{} +// NewProofOp returns a ProofOp for a SparseMerkleProof. func NewProofOp(root, key []byte, hasher HasherType, proof smt.SparseMerkleProof) *ProofOp { return &ProofOp{ Root: root, From ce3ff5545cbea5ce04be50a30de89b23840fb7b5 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 27 Aug 2021 00:55:13 +0800 Subject: [PATCH 06/36] update go.mod - smt --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8cb58692c56f..46023f8a7120 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/improbable-eng/grpc-web v0.14.1 github.com/jhump/protoreflect v1.10.1 github.com/kr/text v0.2.0 // indirect - github.com/lazyledger/smt v0.2.0 // indirect + github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554 github.com/magiconair/properties v1.8.5 github.com/mattn/go-isatty v0.0.14 github.com/onsi/ginkgo v1.16.4 // indirect From 94d1a2e1c0390d5d410d9742117376ec7ddc9a97 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 23 Jun 2021 16:44:42 +0800 Subject: [PATCH 07/36] Refactor SMT, move to store/v2 * Refactors smt to BasicKVStore, moves hashing/decoupling out * Create store/v2; move smt store & refactor w/ new DB --- db/go.mod | 4 +- go.sum | 4 +- store/rootmulti/proof.go | 2 +- store/smt/iterator.go | 101 --------------- store/smt/iterator_test.go | 113 ----------------- store/smt/store.go | 211 ------------------------------- store/smt/store_test.go | 43 ------- store/types/store.go | 12 +- store/{ => v2}/smt/proof.go | 0 store/{ => v2}/smt/proof_test.go | 6 +- store/v2/smt/store.go | 85 +++++++++++++ store/v2/smt/store_test.go | 20 +++ store/v2/types.go | 163 ++++++++++++++++++++++++ store/v2/utils.go | 17 +++ 14 files changed, 301 insertions(+), 480 deletions(-) delete mode 100644 store/smt/iterator.go delete mode 100644 store/smt/iterator_test.go delete mode 100644 store/smt/store.go delete mode 100644 store/smt/store_test.go rename store/{ => v2}/smt/proof.go (100%) rename store/{ => v2}/smt/proof_test.go (90%) create mode 100644 store/v2/smt/store.go create mode 100644 store/v2/smt/store_test.go create mode 100644 store/v2/types.go create mode 100644 store/v2/utils.go diff --git a/db/go.mod b/db/go.mod index 0577e38ffe87..5f35591230fe 100644 --- a/db/go.mod +++ b/db/go.mod @@ -1,7 +1,7 @@ -go 1.17 - module github.com/cosmos/cosmos-sdk/db +go 1.17 + require ( github.com/dgraph-io/badger/v3 v3.2103.1 github.com/google/btree v1.0.0 diff --git a/go.sum b/go.sum index 925c4aca6b6e..f76d2f434290 100644 --- a/go.sum +++ b/go.sum @@ -536,12 +536,12 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554 h1:nDOkLO7klmnEw1s4AyKt1Arvpgyh33uj1JmkYlJaDsk= +github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554/go.mod h1:9+Pb2/tg1PvEgW7aFx4bFhDE4bvbI03zuJ8kb7nJ9Jc= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lazyledger/smt v0.1.1 h1:EiZZnov3ixjvqBYvlPqBqkunarm0wU0tJhWdJxCgbpA= -github.com/lazyledger/smt v0.1.1/go.mod h1:9+Pb2/tg1PvEgW7aFx4bFhDE4bvbI03zuJ8kb7nJ9Jc= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= diff --git a/store/rootmulti/proof.go b/store/rootmulti/proof.go index 6766779b2dd5..d71e8c1adc4f 100644 --- a/store/rootmulti/proof.go +++ b/store/rootmulti/proof.go @@ -3,8 +3,8 @@ package rootmulti import ( "github.com/tendermint/tendermint/crypto/merkle" - "github.com/cosmos/cosmos-sdk/store/smt" storetypes "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/store/v2/smt" ) // RequireProof returns whether proof is required for the subpath. diff --git a/store/smt/iterator.go b/store/smt/iterator.go deleted file mode 100644 index 459460d77532..000000000000 --- a/store/smt/iterator.go +++ /dev/null @@ -1,101 +0,0 @@ -package smt - -import ( - "bytes" - - dbm "github.com/tendermint/tm-db" -) - -type Iterator struct { - store *Store - iter dbm.Iterator -} - -func indexKey(key []byte) []byte { - return append(indexPrefix, key...) -} - -func plainKey(key []byte) []byte { - return key[prefixLen:] -} - -func startKey(key []byte) []byte { - if key == nil { - return dataPrefix - } - return dataKey(key) -} - -func endKey(key []byte) []byte { - if key == nil { - return indexPrefix - } - return dataKey(key) -} - -func newIterator(s *Store, start, end []byte, reverse bool) (*Iterator, error) { - start = startKey(start) - end = endKey(end) - var i dbm.Iterator - var err error - if reverse { - i, err = s.db.ReverseIterator(start, end) - } else { - i, err = s.db.Iterator(start, end) - } - if err != nil { - return nil, err - } - return &Iterator{store: s, iter: i}, nil -} - -// Domain returns the start (inclusive) and end (exclusive) limits of the iterator. -// CONTRACT: start, end readonly []byte -func (i *Iterator) Domain() (start []byte, end []byte) { - start, end = i.iter.Domain() - if bytes.Equal(start, dataPrefix) { - start = nil - } else { - start = plainKey(start) - } - if bytes.Equal(end, indexPrefix) { - end = nil - } else { - end = plainKey(end) - } - return start, end -} - -// Valid returns whether the current iterator is valid. Once invalid, the Iterator remains -// invalid forever. -func (i *Iterator) Valid() bool { - return i.iter.Valid() -} - -// Next moves the iterator to the next key in the database, as defined by order of iteration. -// If Valid returns false, this method will panic. -func (i *Iterator) Next() { - i.iter.Next() -} - -// Key returns the key at the current position. Panics if the iterator is invalid. -// CONTRACT: key readonly []byte -func (i *Iterator) Key() (key []byte) { - return plainKey(i.iter.Key()) -} - -// Value returns the value at the current position. Panics if the iterator is invalid. -// CONTRACT: value readonly []byte -func (i *Iterator) Value() (value []byte) { - return i.store.Get(i.Key()) -} - -// Error returns the last error encountered by the iterator, if any. -func (i *Iterator) Error() error { - return i.iter.Error() -} - -// Close closes the iterator, relasing any allocated resources. -func (i *Iterator) Close() error { - return i.iter.Close() -} diff --git a/store/smt/iterator_test.go b/store/smt/iterator_test.go deleted file mode 100644 index 6a724e665a1e..000000000000 --- a/store/smt/iterator_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package smt_test - -import ( - "bytes" - "sort" - "testing" - - "github.com/cosmos/cosmos-sdk/store/smt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" -) - -func TestIteration(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - pairs := []struct{ key, val []byte }{ - {[]byte("foo"), []byte("bar")}, - {[]byte("lorem"), []byte("ipsum")}, - {[]byte("alpha"), []byte("beta")}, - {[]byte("gamma"), []byte("delta")}, - {[]byte("epsilon"), []byte("zeta")}, - {[]byte("eta"), []byte("theta")}, - {[]byte("iota"), []byte("kappa")}, - } - - s := smt.NewStore(dbm.NewMemDB()) - - for _, p := range pairs { - s.Set(p.key, p.val) - } - - // sort test data by key, to get "expected" ordering - sort.Slice(pairs, func(i, j int) bool { - return bytes.Compare(pairs[i].key, pairs[j].key) < 0 - }) - - iter := s.Iterator([]byte("alpha"), []byte("omega")) - for _, p := range pairs { - require.True(iter.Valid()) - require.Equal(p.key, iter.Key()) - require.Equal(p.val, iter.Value()) - iter.Next() - } - assert.False(iter.Valid()) - assert.NoError(iter.Error()) - assert.NoError(iter.Close()) - - iter = s.Iterator(nil, nil) - for _, p := range pairs { - require.True(iter.Valid()) - require.Equal(p.key, iter.Key()) - require.Equal(p.val, iter.Value()) - iter.Next() - } - assert.False(iter.Valid()) - assert.NoError(iter.Error()) - assert.NoError(iter.Close()) - - iter = s.Iterator([]byte("epsilon"), []byte("gamma")) - for _, p := range pairs[1:4] { - require.True(iter.Valid()) - require.Equal(p.key, iter.Key()) - require.Equal(p.val, iter.Value()) - iter.Next() - } - assert.False(iter.Valid()) - assert.NoError(iter.Error()) - assert.NoError(iter.Close()) - - rIter := s.ReverseIterator(nil, nil) - for i := len(pairs) - 1; i >= 0; i-- { - require.True(rIter.Valid()) - require.Equal(pairs[i].key, rIter.Key()) - require.Equal(pairs[i].val, rIter.Value()) - rIter.Next() - } - assert.False(rIter.Valid()) - assert.NoError(rIter.Error()) - assert.NoError(rIter.Close()) - - // delete something, and ensure that iteration still works - s.Delete([]byte("eta")) - - iter = s.Iterator(nil, nil) - for _, p := range pairs { - if !bytes.Equal([]byte("eta"), p.key) { - require.True(iter.Valid()) - require.Equal(p.key, iter.Key()) - require.Equal(p.val, iter.Value()) - iter.Next() - } - } - assert.False(iter.Valid()) - assert.NoError(iter.Error()) - assert.NoError(iter.Close()) -} - -func TestDomain(t *testing.T) { - assert := assert.New(t) - s := smt.NewStore(dbm.NewMemDB()) - - iter := s.Iterator(nil, nil) - start, end := iter.Domain() - assert.Nil(start) - assert.Nil(end) - - iter = s.Iterator([]byte("foo"), []byte("bar")) - start, end = iter.Domain() - assert.Equal([]byte("foo"), start) - assert.Equal([]byte("bar"), end) -} diff --git a/store/smt/store.go b/store/smt/store.go deleted file mode 100644 index b70b097515d9..000000000000 --- a/store/smt/store.go +++ /dev/null @@ -1,211 +0,0 @@ -package smt - -import ( - "crypto/sha256" - "encoding/binary" - "io" - "sync" - "time" - - "github.com/cosmos/cosmos-sdk/store/cachekv" - "github.com/cosmos/cosmos-sdk/store/tracekv" - "github.com/cosmos/cosmos-sdk/store/types" - "github.com/cosmos/cosmos-sdk/telemetry" - abci "github.com/tendermint/tendermint/abci/types" - dbm "github.com/tendermint/tm-db" - - "github.com/lazyledger/smt" -) - -var ( - _ types.KVStore = (*Store)(nil) - _ types.CommitStore = (*Store)(nil) - _ types.CommitKVStore = (*Store)(nil) - _ types.Queryable = (*Store)(nil) - _ types.StoreWithInitialVersion = (*Store)(nil) -) - -var ( - prefixLen = 1 - versionsPrefix = []byte{0} - dataPrefix = []byte{1} - indexPrefix = []byte{2} - afterIndex = []byte{3} -) - -// Store Implements types.KVStore and CommitKVStore. -type Store struct { - tree *smt.SparseMerkleTree - db dbm.DB - - version int64 - - opts struct { - initialVersion int64 - pruningOptions types.PruningOptions - } - - mtx sync.RWMutex -} - -func NewStore(underlyingDB dbm.DB) *Store { - return &Store{ - tree: smt.NewSparseMerkleTree(underlyingDB, sha256.New()), - db: underlyingDB, - } -} - -// KVStore interface below: - -func (s *Store) GetStoreType() types.StoreType { - return types.StoreTypeSMT -} - -// CacheWrap branches a store. -func (s *Store) CacheWrap() types.CacheWrap { - return cachekv.NewStore(s) -} - -// CacheWrapWithTrace branches a store with tracing enabled. -func (s *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.CacheWrap { - return cachekv.NewStore(tracekv.NewStore(s, w, tc)) -} - -// Get returns nil iff key doesn't exist. Panics on nil key. -func (s *Store) Get(key []byte) []byte { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "get") - val, err := s.db.Get(dataKey(key)) - if err != nil { - panic(err) - } - return val -} - -// Has checks if a key exists. Panics on nil key. -func (s *Store) Has(key []byte) bool { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "has") - has, err := s.db.Has(dataKey(key)) - return err == nil && has -} - -// Set sets the key. Panics on nil key or value. -func (s *Store) Set(key []byte, value []byte) { - kvHash := sha256.Sum256(append(key, value...)) - - s.mtx.Lock() - defer s.mtx.Unlock() - - err := s.db.Set(dataKey(key), value) - if err != nil { - panic(err.Error()) - } - err = s.db.Set(indexKey(kvHash[:]), key) - if err != nil { - panic(err.Error()) - } - _, err = s.tree.Update(key, kvHash[:]) - if err != nil { - panic(err.Error()) - } -} - -// Delete deletes the key. Panics on nil key. -func (s *Store) Delete(key []byte) { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "delete") - - s.mtx.Lock() - defer s.mtx.Unlock() - - _, _ = s.tree.Delete(key) - - dKey := dataKey(key) - defer func() { - _ = s.db.Delete(dKey) - }() - - value, err := s.db.Get(dKey) - if err != nil { - panic(err.Error()) - } - kvHash := sha256.Sum256(append(key, value...)) - _ = s.db.Delete(indexKey(kvHash[:])) -} - -// Iterator over a domain of keys in ascending order. End is exclusive. -// Start must be less than end, or the Iterator is invalid. -// Iterator must be closed by caller. -// To iterate over entire domain, use store.Iterator(nil, nil) -// CONTRACT: No writes may happen within a domain while an iterator exists over it. -// Exceptionally allowed for cachekv.Store, safe to write in the modules. -func (s *Store) Iterator(start []byte, end []byte) types.Iterator { - iter, err := newIterator(s, start, end, false) - if err != nil { - panic(err.Error()) - } - return iter -} - -// Iterator over a domain of keys in descending order. End is exclusive. -// Start must be less than end, or the Iterator is invalid. -// Iterator must be closed by caller. -// CONTRACT: No writes may happen within a domain while an iterator exists over it. -// Exceptionally allowed for cachekv.Store, safe to write in the modules. -func (s *Store) ReverseIterator(start []byte, end []byte) types.Iterator { - iter, err := newIterator(s, start, end, true) - if err != nil { - panic(err.Error()) - } - return iter -} - -// CommitStore interface below: - -func (s *Store) Commit() types.CommitID { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "commit") - version := s.version + 1 - - if version == 1 && s.opts.initialVersion != 0 { - version = s.opts.initialVersion - } - - s.version = version - - b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, uint64(version)) - s.db.Set(append(versionsPrefix, b...), s.tree.Root()) - - return s.LastCommitID() -} - -func (s *Store) LastCommitID() types.CommitID { - return types.CommitID{ - Version: s.version, - Hash: s.tree.Root(), - } -} - -func (s *Store) SetPruning(p types.PruningOptions) { - s.opts.pruningOptions = p -} - -func (s *Store) GetPruning() types.PruningOptions { - return s.opts.pruningOptions -} - -// Queryable interface below: - -func (s *Store) Query(_ abci.RequestQuery) abci.ResponseQuery { - panic("not implemented") -} - -// StoreWithInitialVersion interface below: - -// SetInitialVersion sets the initial version of the SMT tree. It is used when -// starting a new chain at an arbitrary height. -func (s *Store) SetInitialVersion(version int64) { - s.opts.initialVersion = version -} - -func dataKey(key []byte) []byte { - return append(dataPrefix, key...) -} diff --git a/store/smt/store_test.go b/store/smt/store_test.go deleted file mode 100644 index a655e4b6d320..000000000000 --- a/store/smt/store_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package smt_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/cosmos/cosmos-sdk/store/smt" - dbm "github.com/tendermint/tm-db" -) - -func TestVersioning(t *testing.T) { - s := smt.NewStore(dbm.NewMemDB()) - expectedVersion := int64(0) - - s.Set([]byte("foo"), []byte("bar")) - cid1 := s.Commit() - expectedVersion++ - - assert.Equal(t, expectedVersion, cid1.Version) - assert.NotEmpty(t, cid1.Hash) - - s.Set([]byte("foobar"), []byte("baz")) - cid2 := s.Commit() - expectedVersion++ - - assert.Equal(t, expectedVersion, cid2.Version) - assert.NotEmpty(t, cid2.Hash) - assert.NotEqual(t, cid1.Hash, cid2.Hash) -} - -func TestInitialVersion(t *testing.T) { - s := smt.NewStore(dbm.NewMemDB()) - expectedVersion := int64(42) - - s.SetInitialVersion(expectedVersion) - - s.Set([]byte("foo"), []byte("foobar")) - cid := s.Commit() - - assert.Equal(t, expectedVersion, cid.Version) - assert.NotEmpty(t, cid.Hash) -} diff --git a/store/types/store.go b/store/types/store.go index 4e18c43a3b31..11024c88857b 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -186,10 +186,8 @@ type CommitMultiStore interface { //---------subsp------------------------------- // KVStore -// KVStore is a simple interface to get/set data -type KVStore interface { - Store - +// BasicKVStore is a simple interface to get/set data +type BasicKVStore interface { // Get returns nil iff key doesn't exist. Panics on nil key. Get(key []byte) []byte @@ -201,6 +199,12 @@ type KVStore interface { // Delete deletes the key. Panics on nil key. Delete(key []byte) +} + +// KVStore additionally provides iteration and deletion +type KVStore interface { + Store + BasicKVStore // Iterator over a domain of keys in ascending order. End is exclusive. // Start must be less than end, or the Iterator is invalid. diff --git a/store/smt/proof.go b/store/v2/smt/proof.go similarity index 100% rename from store/smt/proof.go rename to store/v2/smt/proof.go diff --git a/store/smt/proof_test.go b/store/v2/smt/proof_test.go similarity index 90% rename from store/smt/proof_test.go rename to store/v2/smt/proof_test.go index 75e7974bc214..1c3f50dd8e99 100644 --- a/store/smt/proof_test.go +++ b/store/v2/smt/proof_test.go @@ -7,14 +7,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - smtstore "github.com/cosmos/cosmos-sdk/store/smt" + "github.com/cosmos/cosmos-sdk/db/memdb" + smtstore "github.com/cosmos/cosmos-sdk/store/v2/smt" "github.com/lazyledger/smt" - dbm "github.com/tendermint/tm-db" ) func TestProofOpInterface(t *testing.T) { hasher := sha256.New() - tree := smt.NewSparseMerkleTree(dbm.NewMemDB(), hasher) + tree := smt.NewSparseMerkleTree(memdb.NewDB(), memdb.NewDB(), hasher) key := []byte("foo") value := []byte("bar") root, err := tree.Update(key, value) diff --git a/store/v2/smt/store.go b/store/v2/smt/store.go new file mode 100644 index 000000000000..91ea52009bfb --- /dev/null +++ b/store/v2/smt/store.go @@ -0,0 +1,85 @@ +package smt + +import ( + "crypto/sha256" + "time" + + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + + "github.com/lazyledger/smt" +) + +var ( + _ types.BasicKVStore = (*Store)(nil) +) + +// Store Implements types.KVStore and CommitKVStore. +type Store struct { + tree *smt.SparseMerkleTree +} + +func NewStore(nodes, values smt.MapStore) *Store { + return &Store{ + tree: smt.NewSparseMerkleTree(nodes, values, sha256.New()), + } +} + +func LoadStore(nodes, values smt.MapStore, root []byte) *Store { + return &Store{ + tree: smt.ImportSparseMerkleTree(nodes, values, sha256.New(), root), + } +} + +func (s *Store) GetProof(key []byte) (*tmcrypto.ProofOps, error) { + proof, err := s.tree.Prove(key) + if err != nil { + return nil, err + } + op := NewProofOp(s.tree.Root(), key, SHA256, proof) + return &tmcrypto.ProofOps{Ops: []tmcrypto.ProofOp{op.ProofOp()}}, nil +} + +func (s *Store) Root() []byte { return s.tree.Root() } + +// BasicKVStore interface below: + +// Get returns nil iff key doesn't exist. Panics on nil key. +func (s *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "get") + val, err := s.tree.Get(key) + if err != nil { + panic(err) + } + return val +} + +// Has checks if a key exists. Panics on nil key. +func (s *Store) Has(key []byte) bool { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "has") + has, err := s.tree.Has(key) + if err != nil { + panic(err) + } + return has +} + +// Set sets the key. Panics on nil key or value. +func (s *Store) Set(key []byte, value []byte) { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "set") + _, err := s.tree.Update(key, value) + if err != nil { + panic(err) + } +} + +// Delete deletes the key. Panics on nil key. +func (s *Store) Delete(key []byte) { + defer telemetry.MeasureSince(time.Now(), "store", "smt", "delete") + + _, err := s.tree.Delete(key) + if err != nil { + panic(err) + } +} diff --git a/store/v2/smt/store_test.go b/store/v2/smt/store_test.go new file mode 100644 index 000000000000..410fc4007e5d --- /dev/null +++ b/store/v2/smt/store_test.go @@ -0,0 +1,20 @@ +package smt_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + store "github.com/cosmos/cosmos-sdk/store/v2/smt" + "github.com/lazyledger/smt" +) + +func TestGetSetHasDelete(t *testing.T) { + s := store.NewStore(smt.NewSimpleMap(), smt.NewSimpleMap()) + + s.Set([]byte("foo"), []byte("bar")) + assert.Equal(t, []byte("bar"), s.Get([]byte("foo"))) + assert.Equal(t, true, s.Has([]byte("foo"))) + s.Delete([]byte("foo")) + assert.Equal(t, false, s.Has([]byte("foo"))) +} diff --git a/store/v2/types.go b/store/v2/types.go new file mode 100644 index 000000000000..0ccae4de31f5 --- /dev/null +++ b/store/v2/types.go @@ -0,0 +1,163 @@ +package types + +import ( + abci "github.com/tendermint/tendermint/abci/types" + dbm "github.com/tendermint/tm-db" + + v1 "github.com/cosmos/cosmos-sdk/store/types" +) + +type Store interface { + GetStoreType() StoreType + // CacheWrapper +} + +// something that can persist to disk +type Committer = v1.Committer + +// Stores of MultiStore must implement CommitStore. +type CommitStore interface { + Committer + Store +} + +// Queryable allows a Store to expose internal state to the abci.Query +// interface. Multistore can route requests to the proper Store. +// +// This is an optional, but useful extension to any CommitStore +type Queryable interface { + Query(abci.RequestQuery) abci.ResponseQuery +} + +type PruningOptions = v1.PruningOptions +type TraceContext = v1.TraceContext +type StoreKey = v1.StoreKey +type CommitID = v1.CommitID +type WriteListener = v1.WriteListener + +// type KVStorePrefixIterator = v1.KVStorePrefixIterator + +//---------subsp------------------------------- +// KVStore + +// BasicKVStore is a simple interface to get/set data +type BasicKVStore interface { + // Get returns nil iff key doesn't exist. Panics on nil key. + Get(key []byte) []byte + + // Has checks if a key exists. Panics on nil key. + Has(key []byte) bool + + // Set sets the key. Panics on nil key or value. + Set(key, value []byte) + + // Delete deletes the key. Panics on nil key. + Delete(key []byte) +} + +// KVStore additionally provides iteration and deletion +type KVStore interface { + Store + BasicKVStore + + // Iterator over a domain of keys in ascending order. End is exclusive. + // Start must be less than end, or the Iterator is invalid. + // Iterator must be closed by caller. + // To iterate over entire domain, use store.Iterator(nil, nil) + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // Exceptionally allowed for cachekv.Store, safe to write in the modules. + Iterator(start, end []byte) Iterator + + // Iterator over a domain of keys in descending order. End is exclusive. + // Start must be less than end, or the Iterator is invalid. + // Iterator must be closed by caller. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // Exceptionally allowed for cachekv.Store, safe to write in the modules. + ReverseIterator(start, end []byte) Iterator +} + +// Iterator is an alias db's Iterator for convenience. +type Iterator = dbm.Iterator + +// CacheKVStore branches a KVStore and provides read cache functionality. +// After calling .Write() on the CacheKVStore, all previously created +// CacheKVStores on the object expire. +type CacheKVStore interface { + KVStore + + // Writes operations to underlying KVStore + Write() +} + +// CommitKVStore is an interface for MultiStore. +type CommitKVStore interface { + Committer + KVStore +} + +type VersionedKVStore interface { + CommitKVStore + AtVersion(int64) (KVStore, error) + VersionExists(int64) bool +} + +//---------------------------------------- +// CacheWrap + +// CacheWrap is the most appropriate interface for store ephemeral branching and cache. +// For example, IAVLStore.CacheWrap() returns a CacheKVStore. CacheWrap should not return +// a Committer, since Commit ephemeral store make no sense. It can return KVStore, +// HeapStore, SpaceStore, etc. +type CacheWrap interface { + CacheWrapper + // Write syncs with the underlying store. + Write() +} + +type CacheWrapper interface { + // CacheWrap branches a store. + CacheWrap() CacheWrap +} + +//---------------------------------------- +// Store types + +// kind of store +type StoreType int + +const ( + StoreTypeMulti StoreType = iota + StoreTypeDB + StoreTypeIAVL + StoreTypeDecoupled + StoreTypeTransient + StoreTypeMemory + StoreTypeSMT +) + +func (st StoreType) String() string { + switch st { + case StoreTypeMulti: + return "StoreTypeMulti" + + case StoreTypeDB: + return "StoreTypeDB" + + case StoreTypeIAVL: + return "StoreTypeIAVL" + + case StoreTypeDecoupled: + return "StoreTypeDecoupled" + + case StoreTypeTransient: + return "StoreTypeTransient" + + case StoreTypeMemory: + return "StoreTypeMemory" + + case StoreTypeSMT: + return "StoreTypeSMT" + } + + return "unknown store type" +} diff --git a/store/v2/utils.go b/store/v2/utils.go new file mode 100644 index 000000000000..b708c761e378 --- /dev/null +++ b/store/v2/utils.go @@ -0,0 +1,17 @@ +package types + +import ( + v1 "github.com/cosmos/cosmos-sdk/store/types" +) + +var PrefixEndBytes = v1.PrefixEndBytes + +// Iterator over all the keys with a certain prefix in ascending order +func KVStorePrefixIterator(kvs KVStore, prefix []byte) Iterator { + return kvs.Iterator(prefix, PrefixEndBytes(prefix)) +} + +// Iterator over all the keys with a certain prefix in descending order. +func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator { + return kvs.ReverseIterator(prefix, PrefixEndBytes(prefix)) +} From 14643aae041b354b491f7e8a1886b668a3909937 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 8 Sep 2021 17:57:47 +0800 Subject: [PATCH 08/36] revise store/v2 types --- store/types/store.go | 5 ++ store/v2/types.go | 173 +++++++------------------------------------ 2 files changed, 30 insertions(+), 148 deletions(-) diff --git a/store/types/store.go b/store/types/store.go index 11024c88857b..aa04a2447579 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -294,6 +294,8 @@ const ( StoreTypeTransient StoreTypeMemory StoreTypeSMT + StoreTypeDecoupled + StoreTypePersistent = StoreTypeDecoupled ) func (st StoreType) String() string { @@ -315,6 +317,9 @@ func (st StoreType) String() string { case StoreTypeSMT: return "StoreTypeSMT" + + case StoreTypeDecoupled: + return "StoreTypeDecoupled" } return "unknown store type" diff --git a/store/v2/types.go b/store/v2/types.go index 0ccae4de31f5..55a66a04b8d1 100644 --- a/store/v2/types.go +++ b/store/v2/types.go @@ -1,163 +1,40 @@ package types import ( - abci "github.com/tendermint/tendermint/abci/types" - dbm "github.com/tendermint/tm-db" - v1 "github.com/cosmos/cosmos-sdk/store/types" ) -type Store interface { - GetStoreType() StoreType - // CacheWrapper -} - -// something that can persist to disk -type Committer = v1.Committer - -// Stores of MultiStore must implement CommitStore. -type CommitStore interface { - Committer - Store -} - -// Queryable allows a Store to expose internal state to the abci.Query -// interface. Multistore can route requests to the proper Store. -// -// This is an optional, but useful extension to any CommitStore -type Queryable interface { - Query(abci.RequestQuery) abci.ResponseQuery -} - -type PruningOptions = v1.PruningOptions -type TraceContext = v1.TraceContext type StoreKey = v1.StoreKey type CommitID = v1.CommitID -type WriteListener = v1.WriteListener - -// type KVStorePrefixIterator = v1.KVStorePrefixIterator - -//---------subsp------------------------------- -// KVStore - -// BasicKVStore is a simple interface to get/set data -type BasicKVStore interface { - // Get returns nil iff key doesn't exist. Panics on nil key. - Get(key []byte) []byte - - // Has checks if a key exists. Panics on nil key. - Has(key []byte) bool - - // Set sets the key. Panics on nil key or value. - Set(key, value []byte) - - // Delete deletes the key. Panics on nil key. - Delete(key []byte) -} - -// KVStore additionally provides iteration and deletion -type KVStore interface { - Store - BasicKVStore - - // Iterator over a domain of keys in ascending order. End is exclusive. - // Start must be less than end, or the Iterator is invalid. - // Iterator must be closed by caller. - // To iterate over entire domain, use store.Iterator(nil, nil) - // CONTRACT: No writes may happen within a domain while an iterator exists over it. - // Exceptionally allowed for cachekv.Store, safe to write in the modules. - Iterator(start, end []byte) Iterator - - // Iterator over a domain of keys in descending order. End is exclusive. - // Start must be less than end, or the Iterator is invalid. - // Iterator must be closed by caller. - // CONTRACT: No writes may happen within a domain while an iterator exists over it. - // Exceptionally allowed for cachekv.Store, safe to write in the modules. - ReverseIterator(start, end []byte) Iterator -} - -// Iterator is an alias db's Iterator for convenience. -type Iterator = dbm.Iterator - -// CacheKVStore branches a KVStore and provides read cache functionality. -// After calling .Write() on the CacheKVStore, all previously created -// CacheKVStores on the object expire. -type CacheKVStore interface { - KVStore - - // Writes operations to underlying KVStore - Write() -} - -// CommitKVStore is an interface for MultiStore. -type CommitKVStore interface { - Committer - KVStore -} - -type VersionedKVStore interface { - CommitKVStore - AtVersion(int64) (KVStore, error) - VersionExists(int64) bool -} - -//---------------------------------------- -// CacheWrap +type StoreUpgrades = v1.StoreUpgrades +type Iterator = v1.Iterator +type PruningOptions = v1.PruningOptions -// CacheWrap is the most appropriate interface for store ephemeral branching and cache. -// For example, IAVLStore.CacheWrap() returns a CacheKVStore. CacheWrap should not return -// a Committer, since Commit ephemeral store make no sense. It can return KVStore, -// HeapStore, SpaceStore, etc. -type CacheWrap interface { - CacheWrapper - // Write syncs with the underlying store. - Write() -} +type TraceContext = v1.TraceContext +type WriteListener = v1.WriteListener -type CacheWrapper interface { - // CacheWrap branches a store. - CacheWrap() CacheWrap -} +type BasicKVStore = v1.BasicKVStore +type KVStore = v1.KVStore +type Committer = v1.Committer +type CommitKVStore = v1.CommitKVStore +type CacheKVStore = v1.CacheKVStore +type Queryable = v1.Queryable +type CacheWrap = v1.CacheWrap + +var ( + PruneDefault = v1.PruneDefault + PruneEverything = v1.PruneEverything + PruneNothing = v1.PruneNothing +) //---------------------------------------- // Store types -// kind of store -type StoreType int - -const ( - StoreTypeMulti StoreType = iota - StoreTypeDB - StoreTypeIAVL - StoreTypeDecoupled - StoreTypeTransient - StoreTypeMemory - StoreTypeSMT -) - -func (st StoreType) String() string { - switch st { - case StoreTypeMulti: - return "StoreTypeMulti" - - case StoreTypeDB: - return "StoreTypeDB" - - case StoreTypeIAVL: - return "StoreTypeIAVL" - - case StoreTypeDecoupled: - return "StoreTypeDecoupled" - - case StoreTypeTransient: - return "StoreTypeTransient" - - case StoreTypeMemory: - return "StoreTypeMemory" - - case StoreTypeSMT: - return "StoreTypeSMT" - } +type StoreType = v1.StoreType - return "unknown store type" -} +// Valid types +const StoreTypeMemory = v1.StoreTypeMemory +const StoreTypeTransient = v1.StoreTypeTransient +const StoreTypeDecoupled = v1.StoreTypeDecoupled +const StoreTypeSMT = v1.StoreTypeSMT +const StoreTypePersistent = StoreTypeDecoupled From 72b05012349f077a0673720f2a743878ad5e14ef Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 27 Aug 2021 01:12:42 +0800 Subject: [PATCH 09/36] Introduce decoupled.Store * Largely adapted from store/ll-smt branch * Builds on new DB interface * Use separate SS/SC DBs * just use hash(value), which smt computes internally * Adapt tests from iavl store - query, prefix iter * Add pruning tests --- go.mod | 3 + store/v2/decoupled/store.go | 420 +++++++++++++++++++++++++++++++ store/v2/decoupled/store_test.go | 327 ++++++++++++++++++++++++ 3 files changed, 750 insertions(+) create mode 100644 store/v2/decoupled/store.go create mode 100644 store/v2/decoupled/store_test.go diff --git a/go.mod b/go.mod index 46023f8a7120..27d0f0bef810 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/confio/ics23/go v0.6.6 github.com/cosmos/btcutil v1.0.4 github.com/cosmos/cosmos-proto v0.0.0-20210914142853-23ed61ac79ce + github.com/cosmos/cosmos-sdk/db v0.0.0 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.17.1 github.com/cosmos/ledger-cosmos-go v0.11.1 @@ -128,3 +129,5 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.33.2 replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 replace github.com/99designs/keyring => github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76 + +replace github.com/cosmos/cosmos-sdk/db => ./db diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go new file mode 100644 index 000000000000..235e01024bb5 --- /dev/null +++ b/store/v2/decoupled/store.go @@ -0,0 +1,420 @@ +package decoupled + +import ( + "crypto/sha256" + "errors" + "fmt" + "sync" + + dbm "github.com/cosmos/cosmos-sdk/db" + "github.com/cosmos/cosmos-sdk/db/prefix" + abci "github.com/tendermint/tendermint/abci/types" + + types "github.com/cosmos/cosmos-sdk/store/v2" + "github.com/cosmos/cosmos-sdk/store/v2/smt" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/types/kv" +) + +var ( + _ types.KVStore = (*Store)(nil) + _ types.CommitKVStore = (*Store)(nil) + _ types.Queryable = (*Store)(nil) +) + +var ( + merkleRootKey = []byte{0} // Key for root hash of Merkle tree + dataPrefix = []byte{1} // Prefix for state mappings + indexPrefix = []byte{2} // Prefix for Store reverse index + merkleNodePrefix = []byte{3} // Prefix for Merkle tree nodes + merkleValuePrefix = []byte{4} // Prefix for Merkle value mappings +) + +var ErrVersionDoesNotExist = errors.New("version does not exist") + +// TODO: +// telemetry + +type StoreConfig struct { + // Version pruning options for backing DBs. + Pruning types.PruningOptions + // The backing DB to use for the state commitment Merkle tree data. + // If nil, Merkle data is stored in the state storage DB under a separate prefix. + MerkleDB dbm.DBConnection + // initialVersion uint64 +} + +// Store is a CommitKVStore which handles state storage and commitments as separate concerns, +// optionally using separate backing key-value DBs for each. +// Allows synchronized R/W access by locking. +type Store struct { + stateDB dbm.DBConnection + stateTxn dbm.DBReadWriter + dataTxn dbm.DBReadWriter + merkleTxn dbm.DBReadWriter + indexTxn dbm.DBReadWriter + // State commitment (SC) KV store for current version + merkleStore *smt.Store + + opts StoreConfig + mtx sync.RWMutex +} + +var DefaultStoreConfig = StoreConfig{Pruning: types.PruneDefault, MerkleDB: nil} + +// NewStore creates a new, empty Store. +func NewStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { + versions, err := db.Versions() + if err != nil { + return nil, err + } + if saved := versions.Count(); saved != 0 { + return nil, fmt.Errorf("DB contains %v existing versions", saved) + } + stateTxn := db.ReadWriter() + merkleTxn := stateTxn + if opts.MerkleDB != nil { + mversions, err := opts.MerkleDB.Versions() + if err != nil { + return nil, err + } + if saved := mversions.Count(); saved != 0 { + return nil, fmt.Errorf("Merkle DB contains %v existing versions", saved) + } + merkleTxn = opts.MerkleDB.ReadWriter() + } + merkleNodes := prefix.NewPrefixReadWriter(merkleTxn, merkleNodePrefix) + merkleValues := prefix.NewPrefixReadWriter(merkleTxn, merkleValuePrefix) + return &Store{ + stateDB: db, + stateTxn: stateTxn, + dataTxn: prefix.NewPrefixReadWriter(stateTxn, dataPrefix), + indexTxn: prefix.NewPrefixReadWriter(stateTxn, indexPrefix), + merkleTxn: merkleTxn, + merkleStore: smt.NewStore(merkleNodes, merkleValues), + opts: opts, + }, nil +} + +// LoadStore loads a Store from a DB. +func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { + stateTxn := db.ReadWriter() + merkleTxn := stateTxn + if opts.MerkleDB != nil { + versions, err := db.Versions() + if err != nil { + return nil, err + } + mversions, err := opts.MerkleDB.Versions() + if err != nil { + return nil, err + } + // Version sets of each DB must match + if !versions.Equal(mversions) { + return nil, fmt.Errorf("Storage and Merkle DB have different version history") + } + merkleTxn = opts.MerkleDB.ReadWriter() + } + root, err := stateTxn.Get(merkleRootKey) + if err != nil { + return nil, err + } + if root == nil { + return nil, fmt.Errorf("could not get root of SMT") + } + return &Store{ + stateDB: db, + stateTxn: stateTxn, + dataTxn: prefix.NewPrefixReadWriter(stateTxn, dataPrefix), + merkleTxn: merkleTxn, + indexTxn: prefix.NewPrefixReadWriter(stateTxn, indexPrefix), + merkleStore: loadSMT(merkleTxn, root), + }, nil +} + +// Access the underlying SMT as a basic KV store +func (s *Store) GetSCStore() types.BasicKVStore { + return s.merkleStore +} + +// Get implements KVStore. +func (s *Store) Get(key []byte) []byte { + s.mtx.RLock() + defer s.mtx.RUnlock() + val, err := s.dataTxn.Get(key) + if err != nil { + panic(err) + } + return val +} + +// Has implements KVStore. +func (s *Store) Has(key []byte) bool { + s.mtx.RLock() + defer s.mtx.RUnlock() + has, err := s.dataTxn.Has(key) + if err != nil { + panic(err) + } + return has +} + +// Set implements KVStore. +func (s *Store) Set(key []byte, value []byte) { + khash := sha256.Sum256(key) + s.mtx.Lock() + defer s.mtx.Unlock() + + err := s.dataTxn.Set(key, value) + if err != nil { + panic(err.Error()) + } + s.merkleStore.Set(key, value) + err = s.indexTxn.Set(khash[:], key) + if err != nil { + panic(err.Error()) + } +} + +// Delete implements KVStore. +func (s *Store) Delete(key []byte) { + khash := sha256.Sum256(key) + s.mtx.Lock() + defer s.mtx.Unlock() + + s.merkleStore.Delete(key) + _ = s.indexTxn.Delete(khash[:]) + _ = s.dataTxn.Delete(key) +} + +type contentsIterator struct { + dbm.Iterator + valid bool +} + +func newIterator(source dbm.Iterator) *contentsIterator { + ret := &contentsIterator{Iterator: source} + ret.Next() + return ret +} + +func (it *contentsIterator) Next() { it.valid = it.Iterator.Next() } +func (it *contentsIterator) Valid() bool { return it.valid } + +// Iterator implements KVStore. +func (s *Store) Iterator(start, end []byte) types.Iterator { + iter, err := s.dataTxn.Iterator(start, end) + if err != nil { + panic(err) + } + return newIterator(iter) +} + +// ReverseIterator implements KVStore. +func (s *Store) ReverseIterator(start, end []byte) types.Iterator { + iter, err := s.dataTxn.ReverseIterator(start, end) + if err != nil { + panic(err) + } + return newIterator(iter) +} + +// GetStoreType implements Store. +func (s *Store) GetStoreType() types.StoreType { + return types.StoreTypeDecoupled +} + +// Commit implements Committer. +func (s *Store) Commit() types.CommitID { + versions, err := s.stateDB.Versions() + if err != nil { + panic(err) + } + cid, err := s.commit(versions.Last() + 1) + if err != nil { + panic(err) + } + + previous := cid.Version - 1 + if s.opts.Pruning.KeepEvery != 1 && s.opts.Pruning.Interval != 0 && cid.Version%int64(s.opts.Pruning.Interval) == 0 { + // The range of newly prunable versions + lastPrunable := previous - int64(s.opts.Pruning.KeepRecent) + firstPrunable := lastPrunable - int64(s.opts.Pruning.Interval) + for version := firstPrunable; version <= lastPrunable; version++ { + if s.opts.Pruning.KeepEvery == 0 || version%int64(s.opts.Pruning.KeepEvery) != 0 { + s.stateDB.DeleteVersion(uint64(version)) + if s.opts.MerkleDB != nil { + s.opts.MerkleDB.DeleteVersion(uint64(version)) + } + } + } + } + return *cid +} + +func (s *Store) commit(target uint64) (*types.CommitID, error) { + root := s.merkleStore.Root() + err := s.stateTxn.Set(merkleRootKey, root) + if err != nil { + return nil, err + } + err = s.stateTxn.Commit() + if err != nil { + return nil, err + } + err = s.stateDB.SaveVersion(target) + if err != nil { + return nil, err + } + + stateTxn := s.stateDB.ReadWriter() + merkleTxn := stateTxn + + // If DBs are not separate, Merkle state has been commmitted & snapshotted + if s.opts.MerkleDB != nil { + // TODO: roll back stateDB save on error? + err = s.merkleTxn.Commit() + if err != nil { + return nil, err + } + err = s.opts.MerkleDB.SaveVersion(target) + if err != nil { + return nil, err + } + merkleTxn = s.opts.MerkleDB.ReadWriter() + } + + s.stateTxn = stateTxn + s.dataTxn = prefix.NewPrefixReadWriter(stateTxn, dataPrefix) + s.indexTxn = prefix.NewPrefixReadWriter(stateTxn, indexPrefix) + s.merkleTxn = merkleTxn + s.merkleStore = loadSMT(merkleTxn, root) + + return &types.CommitID{Version: int64(target), Hash: root}, nil +} + +// LastCommitID implements KVStore. +func (s *Store) LastCommitID() types.CommitID { + versions, err := s.stateDB.Versions() + if err != nil { + panic(err) + } + last := versions.Last() + if last == 0 { + return types.CommitID{} + } + // Latest Merkle root should be the one currently stored + hash, err := s.stateTxn.Get(merkleRootKey) + if err != nil { + panic(err) + } + return types.CommitID{Version: int64(last), Hash: hash} +} + +func (s *Store) GetPruning() types.PruningOptions { return s.opts.Pruning } +func (s *Store) SetPruning(po types.PruningOptions) { s.opts.Pruning = po } +func (s *Store) SetInitialVersion(version int64) {} + +// Query implements ABCI interface, allows queries. +// +// by default we will return from (latest height -1), +// as we will have merkle proofs immediately (header height = data height + 1) +// If latest-1 is not present, use latest (which must be present) +// if you care to have the latest data to see a tx results, you must +// explicitly set the height you want to see +func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { + // defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "query") + + if len(req.Data) == 0 { + return sdkerrors.QueryResult(sdkerrors.Wrap(sdkerrors.ErrTxDecode, "query cannot be zero length"), false) + } + + // if height is 0, use the latest height + height := req.Height + if height == 0 { + versions, err := s.stateDB.Versions() + if err != nil { + return sdkerrors.QueryResult(errors.New("failed to get version info"), false) + } + latest := versions.Last() + if versions.Exists(latest - 1) { + height = int64(latest - 1) + } else { + height = int64(latest) + } + } + res.Height = height + + switch req.Path { + case "/key": + var err error + res.Key = req.Data // data holds the key bytes + + dbr, err := s.stateDB.ReaderAt(uint64(height)) + if err == dbm.ErrVersionDoesNotExist { + return sdkerrors.QueryResult(sdkerrors.ErrInvalidHeight, false) + } + if err != nil { + return sdkerrors.QueryResult(err, false) + } + defer dbr.Discard() + contents := prefix.NewPrefixReader(dbr, dataPrefix) + res.Value, err = contents.Get(res.Key) + if err != nil { + return sdkerrors.QueryResult(sdkerrors.ErrKeyNotFound, false) + } + if !req.Prove { + break + } + merkleView := dbr + if s.opts.MerkleDB != nil { + merkleView, err = s.opts.MerkleDB.ReaderAt(uint64(height)) + if err != nil { + return sdkerrors.QueryResult(fmt.Errorf( + "%w: version does not exist in Merkle DB: %v", sdkerrors.ErrInvalidHeight, height), false) + } + defer merkleView.Discard() + } + root, err := dbr.Get(merkleRootKey) + if err != nil { + return sdkerrors.QueryResult(errors.New("Merkle root hash not found"), false) + } + merkleStore := loadSMT(dbm.ReaderAsReadWriter(merkleView), root) + res.ProofOps, err = merkleStore.GetProof(res.Key) + if err != nil { + return sdkerrors.QueryResult(fmt.Errorf("Merkle proof creation failed for key: %v", res.Key), false) + } + + case "/subspace": + pairs := kv.Pairs{ + Pairs: make([]kv.Pair, 0), + } + + subspace := req.Data + res.Key = subspace + + iterator := s.Iterator(subspace, types.PrefixEndBytes(subspace)) + for ; iterator.Valid(); iterator.Next() { + pairs.Pairs = append(pairs.Pairs, kv.Pair{Key: iterator.Key(), Value: iterator.Value()}) + } + iterator.Close() + + bz, err := pairs.Marshal() + if err != nil { + panic(fmt.Errorf("failed to marshal KV pairs: %w", err)) + } + + res.Value = bz + + default: + return sdkerrors.QueryResult(sdkerrors.Wrapf(sdkerrors.ErrUnknownRequest, "unexpected query path: %v", req.Path), false) + } + + return res +} + +func loadSMT(merkleTxn dbm.DBReadWriter, root []byte) *smt.Store { + merkleNodes := prefix.NewPrefixReadWriter(merkleTxn, merkleNodePrefix) + merkleValues := prefix.NewPrefixReadWriter(merkleTxn, merkleValuePrefix) + return smt.LoadStore(merkleNodes, merkleValues, root) +} diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go new file mode 100644 index 000000000000..331911374b0e --- /dev/null +++ b/store/v2/decoupled/store_test.go @@ -0,0 +1,327 @@ +package decoupled + +import ( + "testing" + + "github.com/stretchr/testify/require" + + abci "github.com/tendermint/tendermint/abci/types" + + dbm "github.com/cosmos/cosmos-sdk/db" + "github.com/cosmos/cosmos-sdk/db/memdb" + types "github.com/cosmos/cosmos-sdk/store/v2" + "github.com/cosmos/cosmos-sdk/types/kv" +) + +var ( + cacheSize = 100 + alohaData = map[string]string{ + "hello": "goodbye", + "aloha": "shalom", + } +) + +func newStoreWithData(t *testing.T, db dbm.DBConnection, storeData map[string]string) *Store { + store, err := NewStore(db, DefaultStoreConfig) + require.NoError(t, err) + + for k, v := range storeData { + store.Set([]byte(k), []byte(v)) + } + return store +} + +func newAlohaStore(t *testing.T, db dbm.DBConnection) *Store { + return newStoreWithData(t, db, alohaData) +} + +func TestGetSetHasDelete(t *testing.T) { + store := newAlohaStore(t, memdb.NewDB()) + key := "hello" + + exists := store.Has([]byte(key)) + require.True(t, exists) + + require.EqualValues(t, []byte(alohaData[key]), store.Get([]byte(key))) + + value2 := "notgoodbye" + store.Set([]byte(key), []byte(value2)) + + require.EqualValues(t, value2, store.Get([]byte(key))) + + store.Delete([]byte(key)) + + exists = store.Has([]byte(key)) + require.False(t, exists) +} + +func TestStoreNoNilSet(t *testing.T) { + store := newAlohaStore(t, memdb.NewDB()) + + require.Panics(t, func() { store.Set(nil, []byte("value")) }, "setting a nil key should panic") + require.Panics(t, func() { store.Set([]byte(""), []byte("value")) }, "setting an empty key should panic") + + require.Panics(t, func() { store.Set([]byte("key"), nil) }, "setting a nil value should panic") +} + +func TestLoadStore(t *testing.T) { + db := memdb.NewDB() + store := newAlohaStore(t, db) + store.Commit() + + store, err := LoadStore(db, DefaultStoreConfig) + require.NoError(t, err) + + value := store.Get([]byte("hello")) + require.Equal(t, []byte("goodbye"), value) +} + +func TestIterators(t *testing.T) { + store := newStoreWithData(t, memdb.NewDB(), map[string]string{ + string([]byte{0x00}): "0", + string([]byte{0x00, 0x00}): "0 0", + string([]byte{0x00, 0x01}): "0 1", + string([]byte{0x00, 0x02}): "0 2", + string([]byte{0x01}): "1", + }) + + var testCase = func(t *testing.T, iter types.Iterator, expected []string) { + var i int + for i = 0; iter.Valid(); iter.Next() { + expectedValue := expected[i] + value := iter.Value() + require.EqualValues(t, string(value), expectedValue) + i++ + } + require.Equal(t, len(expected), i) + } + + testCase(t, store.Iterator(nil, nil), + []string{"0", "0 0", "0 1", "0 2", "1"}) + testCase(t, store.Iterator([]byte{0x00}, nil), + []string{"0", "0 0", "0 1", "0 2", "1"}) + testCase(t, store.Iterator([]byte{0x00}, []byte{0x00, 0x01}), + []string{"0", "0 0"}) + testCase(t, store.Iterator([]byte{0x00}, []byte{0x01}), + []string{"0", "0 0", "0 1", "0 2"}) + testCase(t, store.Iterator([]byte{0x00, 0x01}, []byte{0x01}), + []string{"0 1", "0 2"}) + testCase(t, store.Iterator(nil, []byte{0x01}), + []string{"0", "0 0", "0 1", "0 2"}) + + testCase(t, store.ReverseIterator(nil, nil), + []string{"1", "0 2", "0 1", "0 0", "0"}) + testCase(t, store.ReverseIterator([]byte{0x00}, nil), + []string{"1", "0 2", "0 1", "0 0", "0"}) + testCase(t, store.ReverseIterator([]byte{0x00}, []byte{0x00, 0x01}), + []string{"0 0", "0"}) + testCase(t, store.ReverseIterator([]byte{0x00}, []byte{0x01}), + []string{"0 2", "0 1", "0 0", "0"}) + testCase(t, store.ReverseIterator([]byte{0x00, 0x01}, []byte{0x01}), + []string{"0 2", "0 1"}) + testCase(t, store.ReverseIterator(nil, []byte{0x01}), + []string{"0 2", "0 1", "0 0", "0"}) + + testCase(t, types.KVStorePrefixIterator(store, []byte{0}), + []string{"0", "0 0", "0 1", "0 2"}) + testCase(t, types.KVStoreReversePrefixIterator(store, []byte{0}), + []string{"0 2", "0 1", "0 0", "0"}) +} + +func TestCommit(t *testing.T) { + // Sanity test for Merkle hashing + store := newStoreWithData(t, memdb.NewDB(), nil) + idNew := store.Commit() + store.Set([]byte{0x00}, []byte("a")) + idOne := store.Commit() + require.Equal(t, idNew.Version+1, idOne.Version) + require.NotEqual(t, idNew.Hash, idOne.Hash) + + // Hash of emptied store is same as new store + store.Delete([]byte{0x00}) + idEmptied := store.Commit() + require.Equal(t, idNew.Hash, idEmptied.Hash) + + for i := byte(0); i < 10; i++ { + store.Set([]byte{i}, []byte{i}) + id := store.Commit() + lastid := store.LastCommitID() + require.Equal(t, id.Hash, lastid.Hash) + require.Equal(t, id.Version, lastid.Version) + } +} + +func sliceToSet(slice []uint64) map[uint64]struct{} { + res := make(map[uint64]struct{}) + for _, x := range slice { + res[x] = struct{}{} + } + return res +} + +func TestPruning(t *testing.T) { + // Save versions up to 10 and verify pruning at final commit + testCases := []struct { + keepRecent uint64 + keepEvery uint64 + interval uint64 + kept []uint64 + }{ + {2, 4, 10, []uint64{4, 8, 9, 10}}, + {0, 4, 10, []uint64{4, 8, 10}}, + {0, 0, 10, []uint64{10}}, // everything + {0, 1, 0, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, // nothing + } + + for tci, tc := range testCases { + opts := types.PruningOptions{tc.keepRecent, tc.keepEvery, tc.interval} + db := memdb.NewDB() + store, err := NewStore(db, StoreConfig{Pruning: opts}) + require.NoError(t, err) + + for i := byte(1); i <= 10; i++ { + store.Set([]byte{i}, []byte{i}) + cid := store.Commit() + latest := uint64(i) + require.Equal(t, latest, uint64(cid.Version)) + } + + versions, err := db.Versions() + require.NoError(t, err) + kept := sliceToSet(tc.kept) + for v := uint64(1); v <= 10; v++ { + _, has := kept[v] + require.Equal(t, has, versions.Exists(v), "Version = %v; tc #%d", v, tci) + } + } + + // Test pruning interval + // Save up to 20th version while checking history at specific version checkpoints + opts := types.PruningOptions{0, 5, 10} + testCheckPoints := map[uint64][]uint64{ + 5: []uint64{1, 2, 3, 4, 5}, + 10: []uint64{5, 10}, + 15: []uint64{5, 10, 11, 12, 13, 14, 15}, + 20: []uint64{5, 10, 15, 20}, + } + db := memdb.NewDB() + store, err := NewStore(db, StoreConfig{Pruning: opts}) + require.NoError(t, err) + + for i := byte(1); i <= 20; i++ { + store.Set([]byte{i}, []byte{i}) + cid := store.Commit() + latest := uint64(i) + require.Equal(t, latest, uint64(cid.Version)) + + kept, has := testCheckPoints[latest] + if !has { + continue + } + versions, err := db.Versions() + require.NoError(t, err) + keptMap := sliceToSet(kept) + for v := uint64(1); v <= latest; v++ { + _, has := keptMap[v] + require.Equal(t, has, versions.Exists(v), "Version = %v; tc #%d", v, i) + } + } +} + +func TestQuery(t *testing.T) { + store := newStoreWithData(t, memdb.NewDB(), nil) + + k1, v1 := []byte("key1"), []byte("val1") + k2, v2 := []byte("key2"), []byte("val2") + v3 := []byte("val3") + + ksub := []byte("key") + KVs0 := kv.Pairs{} + KVs1 := kv.Pairs{ + Pairs: []kv.Pair{ + {Key: k1, Value: v1}, + {Key: k2, Value: v2}, + }, + } + KVs2 := kv.Pairs{ + Pairs: []kv.Pair{ + {Key: k1, Value: v3}, + {Key: k2, Value: v2}, + }, + } + + valExpSubEmpty, err := KVs0.Marshal() + require.NoError(t, err) + + valExpSub1, err := KVs1.Marshal() + require.NoError(t, err) + + valExpSub2, err := KVs2.Marshal() + require.NoError(t, err) + + cid := store.Commit() + ver := cid.Version + query := abci.RequestQuery{Path: "/key", Data: k1, Height: ver} + querySub := abci.RequestQuery{Path: "/subspace", Data: ksub, Height: ver} + + // query subspace before anything set + qres := store.Query(querySub) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, valExpSubEmpty, qres.Value) + + // set data + store.Set(k1, v1) + store.Set(k2, v2) + + // set data without commit, doesn't show up + qres = store.Query(query) + require.Equal(t, uint32(0), qres.Code) + require.Nil(t, qres.Value) + + // commit it, but still don't see on old version + cid = store.Commit() + qres = store.Query(query) + require.Equal(t, uint32(0), qres.Code) + require.Nil(t, qres.Value) + + // but yes on the new version + query.Height = cid.Version + qres = store.Query(query) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, v1, qres.Value) + + // and for the subspace + qres = store.Query(querySub) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, valExpSub1, qres.Value) + + // modify + store.Set(k1, v3) + cid = store.Commit() + + // query will return old values, as height is fixed + qres = store.Query(query) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, v1, qres.Value) + + // update to latest in the query and we are happy + query.Height = cid.Version + qres = store.Query(query) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, v3, qres.Value) + query2 := abci.RequestQuery{Path: "/key", Data: k2, Height: cid.Version} + + qres = store.Query(query2) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, v2, qres.Value) + // and for the subspace + qres = store.Query(querySub) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, valExpSub2, qres.Value) + + // default (height 0) will show latest -1 + query0 := abci.RequestQuery{Path: "/key", Data: k1} + qres = store.Query(query0) + require.Equal(t, uint32(0), qres.Code) + require.Equal(t, v1, qres.Value) +} From 9b12233d1d2b9d9865a1f3ed1470027d3bcd937a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 18:47:30 +0800 Subject: [PATCH 10/36] store op telemetry --- store/v2/decoupled/store.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 235e01024bb5..c8c38af2fb91 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sync" + "time" dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/db/prefix" @@ -12,6 +13,7 @@ import ( types "github.com/cosmos/cosmos-sdk/store/v2" "github.com/cosmos/cosmos-sdk/store/v2/smt" + "github.com/cosmos/cosmos-sdk/telemetry" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/types/kv" ) @@ -32,9 +34,6 @@ var ( var ErrVersionDoesNotExist = errors.New("version does not exist") -// TODO: -// telemetry - type StoreConfig struct { // Version pruning options for backing DBs. Pruning types.PruningOptions @@ -139,8 +138,10 @@ func (s *Store) GetSCStore() types.BasicKVStore { // Get implements KVStore. func (s *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "get") s.mtx.RLock() defer s.mtx.RUnlock() + val, err := s.dataTxn.Get(key) if err != nil { panic(err) @@ -150,8 +151,10 @@ func (s *Store) Get(key []byte) []byte { // Has implements KVStore. func (s *Store) Has(key []byte) bool { + defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "has") s.mtx.RLock() defer s.mtx.RUnlock() + has, err := s.dataTxn.Has(key) if err != nil { panic(err) @@ -161,6 +164,7 @@ func (s *Store) Has(key []byte) bool { // Set implements KVStore. func (s *Store) Set(key []byte, value []byte) { + defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "set") khash := sha256.Sum256(key) s.mtx.Lock() defer s.mtx.Unlock() @@ -178,6 +182,7 @@ func (s *Store) Set(key []byte, value []byte) { // Delete implements KVStore. func (s *Store) Delete(key []byte) { + defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "delete") khash := sha256.Sum256(key) s.mtx.Lock() defer s.mtx.Unlock() @@ -323,7 +328,7 @@ func (s *Store) SetInitialVersion(version int64) {} // if you care to have the latest data to see a tx results, you must // explicitly set the height you want to see func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { - // defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "query") + defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "query") if len(req.Data) == 0 { return sdkerrors.QueryResult(sdkerrors.Wrap(sdkerrors.ErrTxDecode, "query cannot be zero length"), false) From ef4b84311481c02c9e1d097b2f81e068c03e0e40 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 19:38:18 +0800 Subject: [PATCH 11/36] rollback commit on failure --- store/v2/decoupled/store.go | 12 +++++++++--- store/v2/decoupled/store_test.go | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index c8c38af2fb91..5d498a738466 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -277,14 +277,20 @@ func (s *Store) commit(target uint64) (*types.CommitID, error) { // If DBs are not separate, Merkle state has been commmitted & snapshotted if s.opts.MerkleDB != nil { - // TODO: roll back stateDB save on error? + rollback := func(e error) error { + if delerr := s.stateDB.DeleteVersion(target); delerr != nil { + return fmt.Errorf("%w: commit rollback failed: %v", e, delerr) + } + return e + } + err = s.merkleTxn.Commit() if err != nil { - return nil, err + return nil, rollback(err) } err = s.opts.MerkleDB.SaveVersion(target) if err != nil { - return nil, err + return nil, rollback(err) } merkleTxn = s.opts.MerkleDB.ReadWriter() } diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 331911374b0e..c999ccca796b 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -1,6 +1,7 @@ package decoupled import ( + "errors" "testing" "github.com/stretchr/testify/require" @@ -128,6 +129,10 @@ func TestIterators(t *testing.T) { []string{"0 2", "0 1", "0 0", "0"}) } +type unsavableDB struct{ *memdb.MemDB } + +func (unsavableDB) SaveVersion(uint64) error { return errors.New("unsavable DB") } + func TestCommit(t *testing.T) { // Sanity test for Merkle hashing store := newStoreWithData(t, memdb.NewDB(), nil) @@ -149,6 +154,16 @@ func TestCommit(t *testing.T) { require.Equal(t, id.Hash, lastid.Hash) require.Equal(t, id.Version, lastid.Version) } + + // Storage commit is rolled back if Merkle commit fails + opts := StoreConfig{MerkleDB: unsavableDB{memdb.NewDB()}, Pruning: types.PruneNothing} + db := memdb.NewDB() + store, err := NewStore(db, opts) + require.NoError(t, err) + require.Panics(t, func() { _ = store.Commit() }) + versions, err := db.Versions() + require.NoError(t, err) + require.Equal(t, 0, versions.Count()) } func sliceToSet(slice []uint64) map[uint64]struct{} { From e498b9be10e2225b32774f8b97b8d49add70241f Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 20:07:52 +0800 Subject: [PATCH 12/36] handle initial version --- store/v2/decoupled/store.go | 25 +++++++++++++++++-------- store/v2/decoupled/store_test.go | 11 +++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 5d498a738466..780869964154 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -39,8 +39,8 @@ type StoreConfig struct { Pruning types.PruningOptions // The backing DB to use for the state commitment Merkle tree data. // If nil, Merkle data is stored in the state storage DB under a separate prefix. - MerkleDB dbm.DBConnection - // initialVersion uint64 + MerkleDB dbm.DBConnection + InitialVersion uint64 } // Store is a CommitKVStore which handles state storage and commitments as separate concerns, @@ -97,13 +97,17 @@ func NewStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { // LoadStore loads a Store from a DB. func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { + versions, err := db.Versions() + if err != nil { + return nil, err + } + if opts.InitialVersion != 0 && versions.Last() < opts.InitialVersion { + return nil, fmt.Errorf("latest saved version is less than initial version: %v < %v", + versions.Last(), opts.InitialVersion) + } stateTxn := db.ReadWriter() merkleTxn := stateTxn if opts.MerkleDB != nil { - versions, err := db.Versions() - if err != nil { - return nil, err - } mversions, err := opts.MerkleDB.Versions() if err != nil { return nil, err @@ -235,7 +239,12 @@ func (s *Store) Commit() types.CommitID { if err != nil { panic(err) } - cid, err := s.commit(versions.Last() + 1) + target := versions.Last() + 1 + // Fast forward to initialversion if needed + if s.opts.InitialVersion != 0 && target < s.opts.InitialVersion { + target = s.opts.InitialVersion + } + cid, err := s.commit(target) if err != nil { panic(err) } @@ -324,7 +333,7 @@ func (s *Store) LastCommitID() types.CommitID { func (s *Store) GetPruning() types.PruningOptions { return s.opts.Pruning } func (s *Store) SetPruning(po types.PruningOptions) { s.opts.Pruning = po } -func (s *Store) SetInitialVersion(version int64) {} +func (s *Store) SetInitialVersion(version int64) { s.opts.InitialVersion = uint64(version) } // Query implements ABCI interface, allows queries. // diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index c999ccca796b..637a4c72917c 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -75,6 +75,11 @@ func TestLoadStore(t *testing.T) { value := store.Get([]byte("hello")) require.Equal(t, []byte("goodbye"), value) + + // Loading with an initial version beyond the lowest should error + opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} + store, err = LoadStore(db, opts) + require.Error(t, err) } func TestIterators(t *testing.T) { @@ -164,6 +169,12 @@ func TestCommit(t *testing.T) { versions, err := db.Versions() require.NoError(t, err) require.Equal(t, 0, versions.Count()) + + opts = StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} + store, err = NewStore(memdb.NewDB(), opts) + require.NoError(t, err) + cid := store.Commit() + require.Equal(t, int64(5), cid.Version) } func sliceToSet(slice []uint64) map[uint64]struct{} { From 456b67f916c2cfa7fc8fc783f952454609c228ba Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 21:06:28 +0800 Subject: [PATCH 13/36] complete store interface --- store/v2/decoupled/store.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 780869964154..59a16c4551ca 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "errors" "fmt" + "io" "sync" "time" @@ -11,6 +12,9 @@ import ( "github.com/cosmos/cosmos-sdk/db/prefix" abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/store/cachekv" + "github.com/cosmos/cosmos-sdk/store/listenkv" + "github.com/cosmos/cosmos-sdk/store/tracekv" types "github.com/cosmos/cosmos-sdk/store/v2" "github.com/cosmos/cosmos-sdk/store/v2/smt" "github.com/cosmos/cosmos-sdk/telemetry" @@ -438,3 +442,15 @@ func loadSMT(merkleTxn dbm.DBReadWriter, root []byte) *smt.Store { merkleValues := prefix.NewPrefixReadWriter(merkleTxn, merkleValuePrefix) return smt.LoadStore(merkleNodes, merkleValues, root) } + +func (st *Store) CacheWrap() types.CacheWrap { + return cachekv.NewStore(st) +} + +func (st *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.CacheWrap { + return cachekv.NewStore(tracekv.NewStore(st, w, tc)) +} + +func (st *Store) CacheWrapWithListeners(storeKey types.StoreKey, listeners []types.WriteListener) types.CacheWrap { + return cachekv.NewStore(listenkv.NewStore(st, storeKey, listeners)) +} From f55855709b822499c94091fe9cc606722660e1fd Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 18:37:05 +0800 Subject: [PATCH 14/36] docs: describe new store package --- docs/core/store.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/core/store.md b/docs/core/store.md index e11c89831397..a3807bf3ee4a 100644 --- a/docs/core/store.md +++ b/docs/core/store.md @@ -222,6 +222,26 @@ When `Store.{Get, Set}()` is called, the store forwards the call to its parent, When `Store.Iterator()` is called, it does not simply prefix the `Store.prefix`, since it does not work as intended. In that case, some of the elements are traversed even they are not starting with the prefix. +## New Store package (`store/v2`) + +The SDK is in the process of transitioning to use the types listed here as the default interface for state storage. At the time of writing, these cannot be used within an application and are not directly compatible with the `CommitMultiStore` and related types. + +### `BasicKVStore` interface + +An interface providing only the basic CRUD functionality (`Get`, `Set`, `Has`, and `Delete` methods), without iteration or caching. This is used to partially expose components of a larger store, such as a `decoupled.Store`. + +### Decoupled Store + +`decoupled.Store` is the new default persistent store, which internally decouples the concerns of state storage and commitment scheme. Values are stored directly in the backing key-value database (the "storage" bucket), while the value's hash is mapped in a separate store which is able to generate a cryptographic commitment (the "state commitment" bucket, implmented with `smt.Store`). + +This can optionally be constructed to use different backend databases for each bucket. + + + +### SMT Store + +A `BasicKVStore` which is used to partially expose functions of an underlying store (for instance, to allow access to the commitment store in `decoupled.Store`). + ## Next {hide} Learn about [encoding](./encoding.md) {hide} From a254ea9879c2d59d33f1942c502a44bbd8ea150b Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Sep 2021 21:47:24 +0800 Subject: [PATCH 15/36] update go.mod --- go.mod | 5 +++-- go.sum | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 27d0f0bef810..ffef908dab96 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.2 // indirect - github.com/dgraph-io/ristretto v0.0.3 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b // indirect @@ -78,6 +78,7 @@ require ( github.com/go-kit/kit v0.10.0 // indirect github.com/go-logfmt/logfmt v0.5.0 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/orderedcode v0.0.1 // indirect @@ -90,7 +91,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d // indirect - github.com/klauspost/compress v1.11.7 // indirect + github.com/klauspost/compress v1.12.3 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 // indirect diff --git a/go.sum b/go.sum index f76d2f434290..82d643527f17 100644 --- a/go.sum +++ b/go.sum @@ -221,9 +221,11 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFM github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= @@ -314,6 +316,7 @@ github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/gateway v1.1.0 h1:u0SuhL9+Il+UbjM9VIE3ntfRujKbvVpFvNB4HbjeVQ0= github.com/gogo/gateway v1.1.0/go.mod h1:S7rR8FRQyG3QFESeSv4l2WnsyzlCLG0CzBbUUo/mbic= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -356,6 +359,7 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -522,8 +526,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= From 78cdaef42977f6c8030320a3af08e5f4d2985fa7 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 28 Sep 2021 13:04:30 +0800 Subject: [PATCH 16/36] error on block height limit --- store/v2/decoupled/store.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 59a16c4551ca..6918f63834bc 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math" "sync" "time" @@ -36,7 +37,10 @@ var ( merkleValuePrefix = []byte{4} // Prefix for Merkle value mappings ) -var ErrVersionDoesNotExist = errors.New("version does not exist") +var ( + ErrVersionDoesNotExist = errors.New("version does not exist") + ErrMaximumHeight = errors.New("maximum block height reached") +) type StoreConfig struct { // Version pruning options for backing DBs. @@ -244,6 +248,9 @@ func (s *Store) Commit() types.CommitID { panic(err) } target := versions.Last() + 1 + if target > math.MaxInt64 { + panic(ErrMaximumHeight) + } // Fast forward to initialversion if needed if s.opts.InitialVersion != 0 && target < s.opts.InitialVersion { target = s.opts.InitialVersion @@ -361,6 +368,9 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { return sdkerrors.QueryResult(errors.New("failed to get version info"), false) } latest := versions.Last() + if latest > math.MaxInt64 { + return sdkerrors.QueryResult(ErrMaximumHeight, false) + } if versions.Exists(latest - 1) { height = int64(latest - 1) } else { From fbd7ced6bf930da7c5fd4de1eac28439c4a2f850 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 28 Sep 2021 15:32:23 +0800 Subject: [PATCH 17/36] remove access to SC store until https://github.com/cosmos/cosmos-sdk/pull/9451 is finalized --- store/v2/decoupled/store.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 6918f63834bc..00a562fec6a5 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -143,11 +143,6 @@ func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { }, nil } -// Access the underlying SMT as a basic KV store -func (s *Store) GetSCStore() types.BasicKVStore { - return s.merkleStore -} - // Get implements KVStore. func (s *Store) Get(key []byte) []byte { defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "get") From d4f8fab3f6b1175a5021bf914e34a23457ad4c7e Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 28 Sep 2021 12:14:42 +0800 Subject: [PATCH 18/36] close txns on error + test case --- internal/util.go | 14 ++++++ store/v2/decoupled/store.go | 78 ++++++++++++++++++++++---------- store/v2/decoupled/store_test.go | 18 ++++++-- 3 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 internal/util.go diff --git a/internal/util.go b/internal/util.go new file mode 100644 index 000000000000..fc67461bd489 --- /dev/null +++ b/internal/util.go @@ -0,0 +1,14 @@ +package util + +import "fmt" + +func CombineErrors(ret error, also error, desc string) error { + if also != nil { + if ret != nil { + ret = fmt.Errorf("%w; %v: %v", ret, desc, also) + } else { + ret = also + } + } + return ret +} diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 00a562fec6a5..da188d07b5dd 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -13,6 +13,7 @@ import ( "github.com/cosmos/cosmos-sdk/db/prefix" abci "github.com/tendermint/tendermint/abci/types" + util "github.com/cosmos/cosmos-sdk/internal" "github.com/cosmos/cosmos-sdk/store/cachekv" "github.com/cosmos/cosmos-sdk/store/listenkv" "github.com/cosmos/cosmos-sdk/store/tracekv" @@ -40,6 +41,7 @@ var ( var ( ErrVersionDoesNotExist = errors.New("version does not exist") ErrMaximumHeight = errors.New("maximum block height reached") + ErrNonEmptyDatabase = errors.New("DB contains saved versions") ) type StoreConfig struct { @@ -70,23 +72,31 @@ type Store struct { var DefaultStoreConfig = StoreConfig{Pruning: types.PruneDefault, MerkleDB: nil} // NewStore creates a new, empty Store. -func NewStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { +// Returns ErrNonEmptyDatabase if a DB contains existing versions. +func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { versions, err := db.Versions() if err != nil { return nil, err } if saved := versions.Count(); saved != 0 { - return nil, fmt.Errorf("DB contains %v existing versions", saved) + return nil, fmt.Errorf("%w: %v existing versions", ErrNonEmptyDatabase, saved) } stateTxn := db.ReadWriter() + defer func() { + if err != nil { + err = util.CombineErrors(err, stateTxn.Discard(), "stateTxn.Discard also failed") + } + }() merkleTxn := stateTxn if opts.MerkleDB != nil { - mversions, err := opts.MerkleDB.Versions() + var mversions dbm.VersionSet + mversions, err = opts.MerkleDB.Versions() if err != nil { - return nil, err + return } if saved := mversions.Count(); saved != 0 { - return nil, fmt.Errorf("Merkle DB contains %v existing versions", saved) + err = fmt.Errorf("%w (Merkle DB): %v existing versions", ErrNonEmptyDatabase, saved) + return } merkleTxn = opts.MerkleDB.ReadWriter() } @@ -104,7 +114,7 @@ func NewStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { } // LoadStore loads a Store from a DB. -func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { +func LoadStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { versions, err := db.Versions() if err != nil { return nil, err @@ -114,24 +124,37 @@ func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { versions.Last(), opts.InitialVersion) } stateTxn := db.ReadWriter() + defer func() { + if err != nil { + err = util.CombineErrors(err, stateTxn.Discard(), "stateTxn.Discard also failed") + } + }() merkleTxn := stateTxn if opts.MerkleDB != nil { - mversions, err := opts.MerkleDB.Versions() + var mversions dbm.VersionSet + mversions, err = opts.MerkleDB.Versions() if err != nil { - return nil, err + return } // Version sets of each DB must match if !versions.Equal(mversions) { - return nil, fmt.Errorf("Storage and Merkle DB have different version history") + err = fmt.Errorf("Storage and Merkle DB have different version history") + return } merkleTxn = opts.MerkleDB.ReadWriter() + defer func() { + if err != nil { + err = util.CombineErrors(err, merkleTxn.Discard(), "merkleTxn.Discard also failed") + } + }() } root, err := stateTxn.Get(merkleRootKey) if err != nil { - return nil, err + return } if root == nil { - return nil, fmt.Errorf("could not get root of SMT") + err = fmt.Errorf("could not get root of SMT") + return } return &Store{ stateDB: db, @@ -143,6 +166,14 @@ func LoadStore(db dbm.DBConnection, opts StoreConfig) (*Store, error) { }, nil } +func (s *Store) Close() error { + err := s.stateTxn.Discard() + if s.opts.MerkleDB != nil { + err = util.CombineErrors(err, s.merkleTxn.Discard(), "merkleTxn.Discard also failed") + } + return err +} + // Get implements KVStore. func (s *Store) Get(key []byte) []byte { defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "get") @@ -272,19 +303,19 @@ func (s *Store) Commit() types.CommitID { return *cid } -func (s *Store) commit(target uint64) (*types.CommitID, error) { +func (s *Store) commit(target uint64) (id *types.CommitID, err error) { root := s.merkleStore.Root() - err := s.stateTxn.Set(merkleRootKey, root) + err = s.stateTxn.Set(merkleRootKey, root) if err != nil { - return nil, err + return } err = s.stateTxn.Commit() if err != nil { - return nil, err + return } err = s.stateDB.SaveVersion(target) if err != nil { - return nil, err + return } stateTxn := s.stateDB.ReadWriter() @@ -292,20 +323,21 @@ func (s *Store) commit(target uint64) (*types.CommitID, error) { // If DBs are not separate, Merkle state has been commmitted & snapshotted if s.opts.MerkleDB != nil { - rollback := func(e error) error { - if delerr := s.stateDB.DeleteVersion(target); delerr != nil { - return fmt.Errorf("%w: commit rollback failed: %v", e, delerr) + defer func() { + if err != nil { + if delerr := s.stateDB.DeleteVersion(target); delerr != nil { + err = fmt.Errorf("%w: commit rollback failed: %v", err, delerr) + } } - return e - } + }() err = s.merkleTxn.Commit() if err != nil { - return nil, rollback(err) + return } err = s.opts.MerkleDB.SaveVersion(target) if err != nil { - return nil, rollback(err) + return } merkleTxn = s.opts.MerkleDB.ReadWriter() } diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 637a4c72917c..cbdb4e484d20 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -65,16 +65,26 @@ func TestStoreNoNilSet(t *testing.T) { require.Panics(t, func() { store.Set([]byte("key"), nil) }, "setting a nil value should panic") } -func TestLoadStore(t *testing.T) { +func TestConstructors(t *testing.T) { db := memdb.NewDB() - store := newAlohaStore(t, db) - store.Commit() + // Fail to load a store from an empty DB store, err := LoadStore(db, DefaultStoreConfig) - require.NoError(t, err) + require.Error(t, err) + store = newAlohaStore(t, db) + store.Commit() + require.NoError(t, store.Close()) + + store, err = LoadStore(db, DefaultStoreConfig) + require.NoError(t, err) value := store.Get([]byte("hello")) require.Equal(t, []byte("goodbye"), value) + require.NoError(t, store.Close()) + + // Fail to create a new store from a non-empty DB + store, err = NewStore(db, DefaultStoreConfig) + require.True(t, errors.Is(err, ErrNonEmptyDatabase)) // Loading with an initial version beyond the lowest should error opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} From f83869c0c5ea7682f7b80aff7e602df768a7144a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 29 Sep 2021 20:18:35 +0800 Subject: [PATCH 19/36] store nits --- store/v2/decoupled/store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index da188d07b5dd..4b804be5ed22 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -209,12 +209,12 @@ func (s *Store) Set(key []byte, value []byte) { err := s.dataTxn.Set(key, value) if err != nil { - panic(err.Error()) + panic(err) } s.merkleStore.Set(key, value) err = s.indexTxn.Set(khash[:], key) if err != nil { - panic(err.Error()) + panic(err) } } @@ -351,7 +351,7 @@ func (s *Store) commit(target uint64) (id *types.CommitID, err error) { return &types.CommitID{Version: int64(target), Hash: root}, nil } -// LastCommitID implements KVStore. +// LastCommitID implements Committer. func (s *Store) LastCommitID() types.CommitID { versions, err := s.stateDB.Versions() if err != nil { @@ -361,7 +361,7 @@ func (s *Store) LastCommitID() types.CommitID { if last == 0 { return types.CommitID{} } - // Latest Merkle root should be the one currently stored + // Latest Merkle root is the one currently stored hash, err := s.stateTxn.Get(merkleRootKey) if err != nil { panic(err) From 41d3da1f975f708ac8f4b0b7208786d17e5faf1d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 29 Sep 2021 22:37:31 +0800 Subject: [PATCH 20/36] store nit --- store/v2/decoupled/store_test.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index cbdb4e484d20..e093b8bffbc0 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -198,21 +198,18 @@ func sliceToSet(slice []uint64) map[uint64]struct{} { func TestPruning(t *testing.T) { // Save versions up to 10 and verify pruning at final commit testCases := []struct { - keepRecent uint64 - keepEvery uint64 - interval uint64 - kept []uint64 + types.PruningOptions + kept []uint64 }{ - {2, 4, 10, []uint64{4, 8, 9, 10}}, - {0, 4, 10, []uint64{4, 8, 10}}, - {0, 0, 10, []uint64{10}}, // everything - {0, 1, 0, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, // nothing + {types.PruningOptions{2, 4, 10}, []uint64{4, 8, 9, 10}}, + {types.PruningOptions{0, 4, 10}, []uint64{4, 8, 10}}, + {types.PruneEverything, []uint64{10}}, + {types.PruneNothing, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, } for tci, tc := range testCases { - opts := types.PruningOptions{tc.keepRecent, tc.keepEvery, tc.interval} db := memdb.NewDB() - store, err := NewStore(db, StoreConfig{Pruning: opts}) + store, err := NewStore(db, StoreConfig{Pruning: tc.PruningOptions}) require.NoError(t, err) for i := byte(1); i <= 10; i++ { From e83d6ddc403b3df49a69f48bd7ab540e113127a0 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 5 Oct 2021 00:10:03 +0800 Subject: [PATCH 21/36] vmgr patches rm unused func fix init version on new test cleanup --- db/version_manager.go | 13 +++---------- db/version_manager_test.go | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/db/version_manager.go b/db/version_manager.go index 1c04d2297ee8..b884e7160114 100644 --- a/db/version_manager.go +++ b/db/version_manager.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "math" ) // VersionManager encapsulates the current valid versions of a DB and computes @@ -18,10 +17,9 @@ var _ VersionSet = (*VersionManager)(nil) func NewVersionManager(versions []uint64) *VersionManager { vmap := make(map[uint64]struct{}) var init, last uint64 - init = math.MaxUint64 for _, ver := range versions { vmap[ver] = struct{}{} - if ver < init { + if init == 0 || ver < init { init = ver } if ver > last { @@ -46,16 +44,11 @@ func (vm *VersionManager) Initial() uint64 { return vm.initial } -func (vm *VersionManager) Next() uint64 { - return vm.Last() + 1 -} - func (vm *VersionManager) Save(target uint64) (uint64, error) { - next := vm.Next() + next := vm.Last() + 1 if target == 0 { target = next - } - if target < next { + } else if target < next { return 0, fmt.Errorf( "target version cannot be less than next sequential version (%v < %v)", target, next) } diff --git a/db/version_manager_test.go b/db/version_manager_test.go index 53e8754a74a2..f78512565c4c 100644 --- a/db/version_manager_test.go +++ b/db/version_manager_test.go @@ -17,10 +17,10 @@ func TestVersionManager(t *testing.T) { require.True(t, vm.Equal(vm)) require.False(t, vm.Exists(0)) - id, err := vm.Save(0) + id1, err := vm.Save(0) require.NoError(t, err) - require.Equal(t, uint64(1), id) - require.True(t, vm.Exists(id)) + require.Equal(t, uint64(1), id1) + require.True(t, vm.Exists(id1)) id2, err := vm.Save(0) require.NoError(t, err) require.True(t, vm.Exists(id2)) @@ -28,17 +28,17 @@ func TestVersionManager(t *testing.T) { require.NoError(t, err) require.True(t, vm.Exists(id3)) - id, err = vm.Save(id) // can't save existing id + _, err = vm.Save(id1) // can't save existing id require.Error(t, err) - id, err = vm.Save(0) + id4, err := vm.Save(0) require.NoError(t, err) - require.True(t, vm.Exists(id)) - vm.Delete(id) - require.False(t, vm.Exists(id)) + require.True(t, vm.Exists(id4)) + vm.Delete(id4) + require.False(t, vm.Exists(id4)) - vm.Delete(1) - require.False(t, vm.Exists(1)) + vm.Delete(id1) + require.False(t, vm.Exists(id1)) require.Equal(t, id2, vm.Initial()) require.Equal(t, id3, vm.Last()) @@ -50,9 +50,9 @@ func TestVersionManager(t *testing.T) { require.Equal(t, []uint64{id2, id3}, all) vmc := vm.Copy() - id, err = vmc.Save(0) + id5, err := vmc.Save(0) require.NoError(t, err) - require.False(t, vm.Exists(id)) // true copy is made + require.False(t, vm.Exists(id5)) // true copy is made vm2 := dbm.NewVersionManager([]uint64{id2, id3}) require.True(t, vm.Equal(vm2)) From 382b21badcde3b57a8108804b4db6300a48c6ec7 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 5 Oct 2021 00:11:46 +0800 Subject: [PATCH 22/36] impl, test store commit rollback --- store/v2/decoupled/store.go | 37 ++++++++++++++- store/v2/decoupled/store_test.go | 80 ++++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 15 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 4b804be5ed22..7423bc60e8ed 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -76,10 +76,15 @@ var DefaultStoreConfig = StoreConfig{Pruning: types.PruneDefault, MerkleDB: nil} func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { versions, err := db.Versions() if err != nil { - return nil, err + return } if saved := versions.Count(); saved != 0 { - return nil, fmt.Errorf("%w: %v existing versions", ErrNonEmptyDatabase, saved) + err = fmt.Errorf("%w: %v existing versions", ErrNonEmptyDatabase, saved) + return + } + err = db.Revert() + if err != nil { + return } stateTxn := db.ReadWriter() defer func() { @@ -98,6 +103,10 @@ func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { err = fmt.Errorf("%w (Merkle DB): %v existing versions", ErrNonEmptyDatabase, saved) return } + err = opts.MerkleDB.Revert() + if err != nil { + return + } merkleTxn = opts.MerkleDB.ReadWriter() } merkleNodes := prefix.NewPrefixReadWriter(merkleTxn, merkleNodePrefix) @@ -123,6 +132,10 @@ func LoadStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { return nil, fmt.Errorf("latest saved version is less than initial version: %v < %v", versions.Last(), opts.InitialVersion) } + err = db.Revert() + if err != nil { + return + } stateTxn := db.ReadWriter() defer func() { if err != nil { @@ -141,6 +154,10 @@ func LoadStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { err = fmt.Errorf("Storage and Merkle DB have different version history") return } + err = opts.MerkleDB.Revert() + if err != nil { + return + } merkleTxn = opts.MerkleDB.ReadWriter() defer func() { if err != nil { @@ -313,12 +330,22 @@ func (s *Store) commit(target uint64) (id *types.CommitID, err error) { if err != nil { return } + defer func() { + if err != nil { + err = util.CombineErrors(err, s.stateDB.Revert(), "stateDB.Revert also failed") + } + }() err = s.stateDB.SaveVersion(target) if err != nil { return } stateTxn := s.stateDB.ReadWriter() + defer func() { + if err != nil { + err = util.CombineErrors(err, stateTxn.Discard(), "stateTxn.Discard also failed") + } + }() merkleTxn := stateTxn // If DBs are not separate, Merkle state has been commmitted & snapshotted @@ -335,6 +362,12 @@ func (s *Store) commit(target uint64) (id *types.CommitID, err error) { if err != nil { return } + defer func() { + if err != nil { + err = util.CombineErrors(err, s.opts.MerkleDB.Revert(), "merkleDB.Revert also failed") + } + }() + err = s.opts.MerkleDB.SaveVersion(target) if err != nil { return diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index e093b8bffbc0..71605b3c93d7 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -148,11 +148,24 @@ type unsavableDB struct{ *memdb.MemDB } func (unsavableDB) SaveVersion(uint64) error { return errors.New("unsavable DB") } +type uncommittableTxn struct{ dbm.DBReadWriter } + +func (tx uncommittableTxn) Commit() error { + tx.Discard() + return errors.New("uncommittable Txn") +} + +type uncommittableDB struct{ *memdb.MemDB } + +func (db uncommittableDB) ReadWriter() dbm.DBReadWriter { + return uncommittableTxn{db.MemDB.ReadWriter()} +} + func TestCommit(t *testing.T) { // Sanity test for Merkle hashing store := newStoreWithData(t, memdb.NewDB(), nil) idNew := store.Commit() - store.Set([]byte{0x00}, []byte("a")) + store.Set([]byte{0}, []byte{0}) idOne := store.Commit() require.Equal(t, idNew.Version+1, idOne.Version) require.NotEqual(t, idNew.Hash, idOne.Hash) @@ -162,26 +175,67 @@ func TestCommit(t *testing.T) { idEmptied := store.Commit() require.Equal(t, idNew.Hash, idEmptied.Hash) - for i := byte(0); i < 10; i++ { + previd := idEmptied + for i := byte(1); i < 5; i++ { store.Set([]byte{i}, []byte{i}) id := store.Commit() lastid := store.LastCommitID() require.Equal(t, id.Hash, lastid.Hash) require.Equal(t, id.Version, lastid.Version) + require.NotEqual(t, previd.Hash, id.Hash) + require.NotEqual(t, previd.Version, id.Version) } - // Storage commit is rolled back if Merkle commit fails - opts := StoreConfig{MerkleDB: unsavableDB{memdb.NewDB()}, Pruning: types.PruneNothing} - db := memdb.NewDB() - store, err := NewStore(db, opts) - require.NoError(t, err) - require.Panics(t, func() { _ = store.Commit() }) - versions, err := db.Versions() - require.NoError(t, err) - require.Equal(t, 0, versions.Count()) + testFailedCommit := func(db dbm.DBConnection, opts StoreConfig) { + store, err := NewStore(db, opts) + require.NoError(t, err) + store.Set([]byte{0}, []byte{0}) + + require.Panics(t, func() { _ = store.Commit() }) + + versions, err := db.Versions() + require.NoError(t, err) + require.Equal(t, 0, versions.Count()) + if opts.MerkleDB != nil { + versions, err = opts.MerkleDB.Versions() + require.NoError(t, err) + require.Equal(t, 0, versions.Count()) + } + + store, err = NewStore(db, opts) + require.NoError(t, err) + require.Nil(t, store.Get([]byte{0})) + } - opts = StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} - store, err = NewStore(memdb.NewDB(), opts) + // Ensure storage commit is rolled back in each failure case + t.Run("recover after failed Commit", func(t *testing.T) { + testFailedCommit( + uncommittableDB{memdb.NewDB()}, + StoreConfig{Pruning: types.PruneNothing}, + ) + }) + t.Run("recover after failed SaveVersion", func(t *testing.T) { + testFailedCommit( + unsavableDB{memdb.NewDB()}, + StoreConfig{Pruning: types.PruneNothing}, + ) + }) + t.Run("recover after failed MerkleDB Commit", func(t *testing.T) { + testFailedCommit( + memdb.NewDB(), + StoreConfig{MerkleDB: uncommittableDB{memdb.NewDB()}, Pruning: types.PruneNothing}, + ) + }) + t.Run("recover after failed MerkleDB SaveVersion", func(t *testing.T) { + testFailedCommit( + memdb.NewDB(), + StoreConfig{MerkleDB: unsavableDB{memdb.NewDB()}, Pruning: types.PruneNothing}, + ) + }) + + // setting initial version + opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} + store, err := NewStore(memdb.NewDB(), opts) require.NoError(t, err) cid := store.Commit() require.Equal(t, int64(5), cid.Version) From a19d6a5a399ff83fe34f03be168b8430ea1ff04a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 6 Oct 2021 23:41:19 +0800 Subject: [PATCH 23/36] remove unneeded files --- store/v2/decoupled/store.go | 2 +- store/v2/decoupled/store_test.go | 2 +- store/v2/types.go | 40 -------------------------------- store/v2/utils.go | 17 -------------- 4 files changed, 2 insertions(+), 59 deletions(-) delete mode 100644 store/v2/types.go delete mode 100644 store/v2/utils.go diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 7423bc60e8ed..14ba69951ea5 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -17,7 +17,7 @@ import ( "github.com/cosmos/cosmos-sdk/store/cachekv" "github.com/cosmos/cosmos-sdk/store/listenkv" "github.com/cosmos/cosmos-sdk/store/tracekv" - types "github.com/cosmos/cosmos-sdk/store/v2" + "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/store/v2/smt" "github.com/cosmos/cosmos-sdk/telemetry" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 71605b3c93d7..a9354cfaded3 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -10,7 +10,7 @@ import ( dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/db/memdb" - types "github.com/cosmos/cosmos-sdk/store/v2" + "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/types/kv" ) diff --git a/store/v2/types.go b/store/v2/types.go deleted file mode 100644 index 55a66a04b8d1..000000000000 --- a/store/v2/types.go +++ /dev/null @@ -1,40 +0,0 @@ -package types - -import ( - v1 "github.com/cosmos/cosmos-sdk/store/types" -) - -type StoreKey = v1.StoreKey -type CommitID = v1.CommitID -type StoreUpgrades = v1.StoreUpgrades -type Iterator = v1.Iterator -type PruningOptions = v1.PruningOptions - -type TraceContext = v1.TraceContext -type WriteListener = v1.WriteListener - -type BasicKVStore = v1.BasicKVStore -type KVStore = v1.KVStore -type Committer = v1.Committer -type CommitKVStore = v1.CommitKVStore -type CacheKVStore = v1.CacheKVStore -type Queryable = v1.Queryable -type CacheWrap = v1.CacheWrap - -var ( - PruneDefault = v1.PruneDefault - PruneEverything = v1.PruneEverything - PruneNothing = v1.PruneNothing -) - -//---------------------------------------- -// Store types - -type StoreType = v1.StoreType - -// Valid types -const StoreTypeMemory = v1.StoreTypeMemory -const StoreTypeTransient = v1.StoreTypeTransient -const StoreTypeDecoupled = v1.StoreTypeDecoupled -const StoreTypeSMT = v1.StoreTypeSMT -const StoreTypePersistent = StoreTypeDecoupled diff --git a/store/v2/utils.go b/store/v2/utils.go deleted file mode 100644 index b708c761e378..000000000000 --- a/store/v2/utils.go +++ /dev/null @@ -1,17 +0,0 @@ -package types - -import ( - v1 "github.com/cosmos/cosmos-sdk/store/types" -) - -var PrefixEndBytes = v1.PrefixEndBytes - -// Iterator over all the keys with a certain prefix in ascending order -func KVStorePrefixIterator(kvs KVStore, prefix []byte) Iterator { - return kvs.Iterator(prefix, PrefixEndBytes(prefix)) -} - -// Iterator over all the keys with a certain prefix in descending order. -func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator { - return kvs.ReverseIterator(prefix, PrefixEndBytes(prefix)) -} From 2a70da1f9a286e21158792d14ba82aaea8047a72 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 12 Oct 2021 16:26:47 +0800 Subject: [PATCH 24/36] decoupled Store - consolidate constructors NewStore handles new/load, no LoadStore --- store/v2/decoupled/store.go | 91 +++++++++----------------------- store/v2/decoupled/store_test.go | 14 ++--- 2 files changed, 28 insertions(+), 77 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 14ba69951ea5..3fe14a8fb710 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -41,7 +41,6 @@ var ( var ( ErrVersionDoesNotExist = errors.New("version does not exist") ErrMaximumHeight = errors.New("maximum block height reached") - ErrNonEmptyDatabase = errors.New("DB contains saved versions") ) type StoreConfig struct { @@ -71,16 +70,20 @@ type Store struct { var DefaultStoreConfig = StoreConfig{Pruning: types.PruneDefault, MerkleDB: nil} -// NewStore creates a new, empty Store. -// Returns ErrNonEmptyDatabase if a DB contains existing versions. +// NewStore creates a new Store, or loads one if db contains existing data. func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { versions, err := db.Versions() if err != nil { return } + loadExisting := false + // If the DB is not empty, attempt to load existing data if saved := versions.Count(); saved != 0 { - err = fmt.Errorf("%w: %v existing versions", ErrNonEmptyDatabase, saved) - return + if opts.InitialVersion != 0 && versions.Last() < opts.InitialVersion { + return nil, fmt.Errorf("latest saved version is less than initial version: %v < %v", + versions.Last(), opts.InitialVersion) + } + loadExisting = true } err = db.Revert() if err != nil { @@ -99,8 +102,9 @@ func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { if err != nil { return } - if saved := mversions.Count(); saved != 0 { - err = fmt.Errorf("%w (Merkle DB): %v existing versions", ErrNonEmptyDatabase, saved) + // Version sets of each DB must match + if !versions.Equal(mversions) { + err = fmt.Errorf("Storage and Merkle DB have different version history") return } err = opts.MerkleDB.Revert() @@ -109,77 +113,32 @@ func NewStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { } merkleTxn = opts.MerkleDB.ReadWriter() } - merkleNodes := prefix.NewPrefixReadWriter(merkleTxn, merkleNodePrefix) - merkleValues := prefix.NewPrefixReadWriter(merkleTxn, merkleValuePrefix) - return &Store{ - stateDB: db, - stateTxn: stateTxn, - dataTxn: prefix.NewPrefixReadWriter(stateTxn, dataPrefix), - indexTxn: prefix.NewPrefixReadWriter(stateTxn, indexPrefix), - merkleTxn: merkleTxn, - merkleStore: smt.NewStore(merkleNodes, merkleValues), - opts: opts, - }, nil -} -// LoadStore loads a Store from a DB. -func LoadStore(db dbm.DBConnection, opts StoreConfig) (ret *Store, err error) { - versions, err := db.Versions() - if err != nil { - return nil, err - } - if opts.InitialVersion != 0 && versions.Last() < opts.InitialVersion { - return nil, fmt.Errorf("latest saved version is less than initial version: %v < %v", - versions.Last(), opts.InitialVersion) - } - err = db.Revert() - if err != nil { - return - } - stateTxn := db.ReadWriter() - defer func() { - if err != nil { - err = util.CombineErrors(err, stateTxn.Discard(), "stateTxn.Discard also failed") - } - }() - merkleTxn := stateTxn - if opts.MerkleDB != nil { - var mversions dbm.VersionSet - mversions, err = opts.MerkleDB.Versions() + var merkleStore *smt.Store + if loadExisting { + var root []byte + root, err = stateTxn.Get(merkleRootKey) if err != nil { return } - // Version sets of each DB must match - if !versions.Equal(mversions) { - err = fmt.Errorf("Storage and Merkle DB have different version history") - return - } - err = opts.MerkleDB.Revert() - if err != nil { + if root == nil { + err = fmt.Errorf("could not get root of SMT") return } - merkleTxn = opts.MerkleDB.ReadWriter() - defer func() { - if err != nil { - err = util.CombineErrors(err, merkleTxn.Discard(), "merkleTxn.Discard also failed") - } - }() - } - root, err := stateTxn.Get(merkleRootKey) - if err != nil { - return - } - if root == nil { - err = fmt.Errorf("could not get root of SMT") - return + merkleStore = loadSMT(merkleTxn, root) + } else { + merkleNodes := prefix.NewPrefixReadWriter(merkleTxn, merkleNodePrefix) + merkleValues := prefix.NewPrefixReadWriter(merkleTxn, merkleValuePrefix) + merkleStore = smt.NewStore(merkleNodes, merkleValues) } return &Store{ stateDB: db, stateTxn: stateTxn, dataTxn: prefix.NewPrefixReadWriter(stateTxn, dataPrefix), - merkleTxn: merkleTxn, indexTxn: prefix.NewPrefixReadWriter(stateTxn, indexPrefix), - merkleStore: loadSMT(merkleTxn, root), + merkleTxn: merkleTxn, + merkleStore: merkleStore, + opts: opts, }, nil } diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index a9354cfaded3..a960d655ccd8 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -68,27 +68,19 @@ func TestStoreNoNilSet(t *testing.T) { func TestConstructors(t *testing.T) { db := memdb.NewDB() - // Fail to load a store from an empty DB - store, err := LoadStore(db, DefaultStoreConfig) - require.Error(t, err) - - store = newAlohaStore(t, db) + store := newAlohaStore(t, db) store.Commit() require.NoError(t, store.Close()) - store, err = LoadStore(db, DefaultStoreConfig) + store, err := NewStore(db, DefaultStoreConfig) require.NoError(t, err) value := store.Get([]byte("hello")) require.Equal(t, []byte("goodbye"), value) require.NoError(t, store.Close()) - // Fail to create a new store from a non-empty DB - store, err = NewStore(db, DefaultStoreConfig) - require.True(t, errors.Is(err, ErrNonEmptyDatabase)) - // Loading with an initial version beyond the lowest should error opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} - store, err = LoadStore(db, opts) + store, err = NewStore(db, opts) require.Error(t, err) } From 20e9fb9d1f55ef468096fa6af2a9864560bde6aa Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 12 Oct 2021 23:43:37 +0800 Subject: [PATCH 25/36] fix test --- store/v2/smt/proof_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/v2/smt/proof_test.go b/store/v2/smt/proof_test.go index 1c3f50dd8e99..94e93edb85ac 100644 --- a/store/v2/smt/proof_test.go +++ b/store/v2/smt/proof_test.go @@ -14,7 +14,7 @@ import ( func TestProofOpInterface(t *testing.T) { hasher := sha256.New() - tree := smt.NewSparseMerkleTree(memdb.NewDB(), memdb.NewDB(), hasher) + tree := smt.NewSparseMerkleTree(memdb.NewDB().ReadWriter(), memdb.NewDB().ReadWriter(), hasher) key := []byte("foo") value := []byte("bar") root, err := tree.Update(key, value) From 97bad68c1c7c09c4f2f02f58f586bec6a0312dc6 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 15 Oct 2021 17:35:05 +0800 Subject: [PATCH 26/36] fill out test coverage and tweak errors --- store/v2/decoupled/store.go | 16 +- store/v2/decoupled/store_test.go | 370 +++++++++++++++++++++++-------- types/errors/abci.go | 2 +- 3 files changed, 282 insertions(+), 106 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 3fe14a8fb710..b4c4c0b2ad32 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -387,14 +387,14 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { return sdkerrors.QueryResult(errors.New("failed to get version info"), false) } latest := versions.Last() - if latest > math.MaxInt64 { - return sdkerrors.QueryResult(ErrMaximumHeight, false) - } if versions.Exists(latest - 1) { height = int64(latest - 1) } else { height = int64(latest) } + if height < 0 { + return sdkerrors.QueryResult(fmt.Errorf("height overflow: %v", latest), false) + } } res.Height = height @@ -404,10 +404,10 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { res.Key = req.Data // data holds the key bytes dbr, err := s.stateDB.ReaderAt(uint64(height)) - if err == dbm.ErrVersionDoesNotExist { - return sdkerrors.QueryResult(sdkerrors.ErrInvalidHeight, false) - } if err != nil { + if err == dbm.ErrVersionDoesNotExist { + err = sdkerrors.ErrInvalidHeight + } return sdkerrors.QueryResult(err, false) } defer dbr.Discard() @@ -423,8 +423,8 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { if s.opts.MerkleDB != nil { merkleView, err = s.opts.MerkleDB.ReaderAt(uint64(height)) if err != nil { - return sdkerrors.QueryResult(fmt.Errorf( - "%w: version does not exist in Merkle DB: %v", sdkerrors.ErrInvalidHeight, height), false) + return sdkerrors.QueryResult( + fmt.Errorf("version exists in state DB but not Merkle DB: %v", height), false) } defer merkleView.Discard() } diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index a960d655ccd8..8a8f913196e2 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -2,6 +2,7 @@ package decoupled import ( "errors" + "math" "testing" "github.com/stretchr/testify/require" @@ -11,6 +12,7 @@ import ( dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/db/memdb" "github.com/cosmos/cosmos-sdk/store/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/types/kv" ) @@ -54,15 +56,17 @@ func TestGetSetHasDelete(t *testing.T) { exists = store.Has([]byte(key)) require.False(t, exists) -} - -func TestStoreNoNilSet(t *testing.T) { - store := newAlohaStore(t, memdb.NewDB()) - require.Panics(t, func() { store.Set(nil, []byte("value")) }, "setting a nil key should panic") - require.Panics(t, func() { store.Set([]byte(""), []byte("value")) }, "setting an empty key should panic") - - require.Panics(t, func() { store.Set([]byte("key"), nil) }, "setting a nil value should panic") + require.Panics(t, func() { store.Get(nil) }, "Get(nil key) should panic") + require.Panics(t, func() { store.Get([]byte{}) }, "Get(empty key) should panic") + require.Panics(t, func() { store.Has(nil) }, "Has(nil key) should panic") + require.Panics(t, func() { store.Has([]byte{}) }, "Has(empty key) should panic") + require.Panics(t, func() { store.Set(nil, []byte("value")) }, "Set(nil key) should panic") + require.Panics(t, func() { store.Set([]byte{}, []byte("value")) }, "Set(empty key) should panic") + require.Panics(t, func() { store.Set([]byte("key"), nil) }, "Set(nil value) should panic") + store.indexTxn = rwCrudFails{store.indexTxn} + require.Panics(t, func() { store.Set([]byte("key"), []byte("value")) }, + "Set() when index fails should panic") } func TestConstructors(t *testing.T) { @@ -82,6 +86,44 @@ func TestConstructors(t *testing.T) { opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} store, err = NewStore(db, opts) require.Error(t, err) + db.Close() + + store, err = NewStore(dbVersionsFails{memdb.NewDB()}, DefaultStoreConfig) + require.Error(t, err) + store, err = NewStore(db, StoreConfig{MerkleDB: dbVersionsFails{memdb.NewDB()}}) + require.Error(t, err) + + // can't use a DB with open writers + db = memdb.NewDB() + merkledb := memdb.NewDB() + w := db.Writer() + store, err = NewStore(db, DefaultStoreConfig) + require.Error(t, err) + w.Discard() + w = merkledb.Writer() + store, err = NewStore(db, StoreConfig{MerkleDB: merkledb}) + require.Error(t, err) + w.Discard() + + // can't use DBs with different version history + merkledb.SaveNextVersion() + store, err = NewStore(db, StoreConfig{MerkleDB: merkledb}) + require.Error(t, err) + merkledb.Close() + + // can't load existing store when we can't access the last commit hash + store, err = NewStore(db, DefaultStoreConfig) + require.NoError(t, err) + store.Commit() + + w = db.Writer() + w.Delete(merkleRootKey) + w.Commit() + store, err = NewStore(db, DefaultStoreConfig) + require.Error(t, err) + + store, err = NewStore(dbRWCrudFails{db}, DefaultStoreConfig) + require.Error(t, err) } func TestIterators(t *testing.T) { @@ -106,131 +148,185 @@ func TestIterators(t *testing.T) { testCase(t, store.Iterator(nil, nil), []string{"0", "0 0", "0 1", "0 2", "1"}) - testCase(t, store.Iterator([]byte{0x00}, nil), + testCase(t, store.Iterator([]byte{0}, nil), []string{"0", "0 0", "0 1", "0 2", "1"}) - testCase(t, store.Iterator([]byte{0x00}, []byte{0x00, 0x01}), + testCase(t, store.Iterator([]byte{0}, []byte{0, 1}), []string{"0", "0 0"}) - testCase(t, store.Iterator([]byte{0x00}, []byte{0x01}), + testCase(t, store.Iterator([]byte{0}, []byte{1}), []string{"0", "0 0", "0 1", "0 2"}) - testCase(t, store.Iterator([]byte{0x00, 0x01}, []byte{0x01}), + testCase(t, store.Iterator([]byte{0, 1}, []byte{1}), []string{"0 1", "0 2"}) - testCase(t, store.Iterator(nil, []byte{0x01}), + testCase(t, store.Iterator(nil, []byte{1}), []string{"0", "0 0", "0 1", "0 2"}) + testCase(t, store.Iterator([]byte{0}, []byte{0}), []string{}) // start = end + testCase(t, store.Iterator([]byte{1}, []byte{0}), []string{}) // start > end testCase(t, store.ReverseIterator(nil, nil), []string{"1", "0 2", "0 1", "0 0", "0"}) - testCase(t, store.ReverseIterator([]byte{0x00}, nil), + testCase(t, store.ReverseIterator([]byte{0}, nil), []string{"1", "0 2", "0 1", "0 0", "0"}) - testCase(t, store.ReverseIterator([]byte{0x00}, []byte{0x00, 0x01}), + testCase(t, store.ReverseIterator([]byte{0}, []byte{0, 1}), []string{"0 0", "0"}) - testCase(t, store.ReverseIterator([]byte{0x00}, []byte{0x01}), + testCase(t, store.ReverseIterator([]byte{0}, []byte{1}), []string{"0 2", "0 1", "0 0", "0"}) - testCase(t, store.ReverseIterator([]byte{0x00, 0x01}, []byte{0x01}), + testCase(t, store.ReverseIterator([]byte{0, 1}, []byte{1}), []string{"0 2", "0 1"}) - testCase(t, store.ReverseIterator(nil, []byte{0x01}), + testCase(t, store.ReverseIterator(nil, []byte{1}), []string{"0 2", "0 1", "0 0", "0"}) + testCase(t, store.ReverseIterator([]byte{0}, []byte{0}), []string{}) // start = end + testCase(t, store.ReverseIterator([]byte{1}, []byte{0}), []string{}) // start > end testCase(t, types.KVStorePrefixIterator(store, []byte{0}), []string{"0", "0 0", "0 1", "0 2"}) testCase(t, types.KVStoreReversePrefixIterator(store, []byte{0}), []string{"0 2", "0 1", "0 0", "0"}) -} - -type unsavableDB struct{ *memdb.MemDB } -func (unsavableDB) SaveVersion(uint64) error { return errors.New("unsavable DB") } - -type uncommittableTxn struct{ dbm.DBReadWriter } - -func (tx uncommittableTxn) Commit() error { - tx.Discard() - return errors.New("uncommittable Txn") + require.Panics(t, func() { store.Iterator([]byte{}, nil) }, "Iterator(empty key) should panic") + require.Panics(t, func() { store.Iterator(nil, []byte{}) }, "Iterator(empty key) should panic") + require.Panics(t, func() { store.ReverseIterator([]byte{}, nil) }, "Iterator(empty key) should panic") + require.Panics(t, func() { store.ReverseIterator(nil, []byte{}) }, "Iterator(empty key) should panic") } -type uncommittableDB struct{ *memdb.MemDB } +func TestCommit(t *testing.T) { + testBasic := func(opts StoreConfig) { + // Sanity test for Merkle hashing + store, err := NewStore(memdb.NewDB(), opts) + require.NoError(t, err) + require.Zero(t, store.LastCommitID()) + idNew := store.Commit() + store.Set([]byte{0}, []byte{0}) + idOne := store.Commit() + require.Equal(t, idNew.Version+1, idOne.Version) + require.NotEqual(t, idNew.Hash, idOne.Hash) -func (db uncommittableDB) ReadWriter() dbm.DBReadWriter { - return uncommittableTxn{db.MemDB.ReadWriter()} -} + // Hash of emptied store is same as new store + store.Delete([]byte{0}) + idEmptied := store.Commit() + require.Equal(t, idNew.Hash, idEmptied.Hash) -func TestCommit(t *testing.T) { - // Sanity test for Merkle hashing - store := newStoreWithData(t, memdb.NewDB(), nil) - idNew := store.Commit() - store.Set([]byte{0}, []byte{0}) - idOne := store.Commit() - require.Equal(t, idNew.Version+1, idOne.Version) - require.NotEqual(t, idNew.Hash, idOne.Hash) - - // Hash of emptied store is same as new store - store.Delete([]byte{0x00}) - idEmptied := store.Commit() - require.Equal(t, idNew.Hash, idEmptied.Hash) - - previd := idEmptied - for i := byte(1); i < 5; i++ { - store.Set([]byte{i}, []byte{i}) - id := store.Commit() - lastid := store.LastCommitID() - require.Equal(t, id.Hash, lastid.Hash) - require.Equal(t, id.Version, lastid.Version) - require.NotEqual(t, previd.Hash, id.Hash) - require.NotEqual(t, previd.Version, id.Version) + previd := idEmptied + for i := byte(1); i < 5; i++ { + store.Set([]byte{i}, []byte{i}) + id := store.Commit() + lastid := store.LastCommitID() + require.Equal(t, id.Hash, lastid.Hash) + require.Equal(t, id.Version, lastid.Version) + require.NotEqual(t, previd.Hash, id.Hash) + require.NotEqual(t, previd.Version, id.Version) + } + } + for _, opts := range []StoreConfig{ + StoreConfig{Pruning: types.PruneNothing}, + StoreConfig{Pruning: types.PruneEverything}, + StoreConfig{Pruning: types.PruneNothing, MerkleDB: memdb.NewDB()}, + StoreConfig{Pruning: types.PruneEverything, MerkleDB: memdb.NewDB()}, + } { + // t.Run(fmt.Sprintf("basic with config %v", i)) + testBasic(opts) } - testFailedCommit := func(db dbm.DBConnection, opts StoreConfig) { - store, err := NewStore(db, opts) - require.NoError(t, err) - store.Set([]byte{0}, []byte{0}) + testFailedCommit := func(t *testing.T, store *Store, db dbm.DBConnection) { + opts := store.opts + if db == nil { + db = store.stateDB + } - require.Panics(t, func() { _ = store.Commit() }) + store.Set([]byte{0}, []byte{0}) + // require.Panics(t, store.Commit) + require.Panics(t, func() { store.Commit() }) + require.NoError(t, store.Close()) - versions, err := db.Versions() - require.NoError(t, err) + versions, _ := db.Versions() require.Equal(t, 0, versions.Count()) if opts.MerkleDB != nil { - versions, err = opts.MerkleDB.Versions() - require.NoError(t, err) + versions, _ = opts.MerkleDB.Versions() require.Equal(t, 0, versions.Count()) } - store, err = NewStore(db, opts) + store, err := NewStore(db, opts) require.NoError(t, err) require.Nil(t, store.Get([]byte{0})) + require.NoError(t, store.Close()) } // Ensure storage commit is rolled back in each failure case t.Run("recover after failed Commit", func(t *testing.T) { - testFailedCommit( - uncommittableDB{memdb.NewDB()}, - StoreConfig{Pruning: types.PruneNothing}, - ) + store, err := NewStore( + dbRWCommitFails{memdb.NewDB()}, + StoreConfig{Pruning: types.PruneNothing}) + require.NoError(t, err) + testFailedCommit(t, store, nil) }) t.Run("recover after failed SaveVersion", func(t *testing.T) { - testFailedCommit( - unsavableDB{memdb.NewDB()}, - StoreConfig{Pruning: types.PruneNothing}, - ) + store, err := NewStore( + dbSaveVersionFails{memdb.NewDB()}, + StoreConfig{Pruning: types.PruneNothing}) + require.NoError(t, err) + testFailedCommit(t, store, nil) }) t.Run("recover after failed MerkleDB Commit", func(t *testing.T) { - testFailedCommit( - memdb.NewDB(), - StoreConfig{MerkleDB: uncommittableDB{memdb.NewDB()}, Pruning: types.PruneNothing}, - ) + store, err := NewStore(memdb.NewDB(), + StoreConfig{MerkleDB: dbRWCommitFails{memdb.NewDB()}, Pruning: types.PruneNothing}) + require.NoError(t, err) + testFailedCommit(t, store, nil) }) t.Run("recover after failed MerkleDB SaveVersion", func(t *testing.T) { - testFailedCommit( - memdb.NewDB(), - StoreConfig{MerkleDB: unsavableDB{memdb.NewDB()}, Pruning: types.PruneNothing}, - ) + store, err := NewStore(memdb.NewDB(), + StoreConfig{MerkleDB: dbSaveVersionFails{memdb.NewDB()}, Pruning: types.PruneNothing}) + require.NoError(t, err) + testFailedCommit(t, store, nil) + }) + + t.Run("stateDB.DeleteVersion error triggers failure", func(t *testing.T) { + store, err := NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) + require.NoError(t, err) + store.merkleTxn = rwCommitFails{store.merkleTxn} + store.stateDB = dbDeleteVersionFails{store.stateDB} + require.Panics(t, func() { store.Commit() }) + }) + t.Run("stateDB.Versions error triggers failure", func(t *testing.T) { + db := memdb.NewDB() + store, err := NewStore(db, DefaultStoreConfig) + require.NoError(t, err) + store.stateDB = dbVersionsFails{store.stateDB} + testFailedCommit(t, store, db) + }) + t.Run("stateTxn.Set error triggers failure", func(t *testing.T) { + store, err := NewStore(memdb.NewDB(), DefaultStoreConfig) + require.NoError(t, err) + store.stateTxn = rwCrudFails{store.stateTxn} + testFailedCommit(t, store, nil) + }) + + t.Run("height overflow triggers failure", func(t *testing.T) { + store, err := NewStore(memdb.NewDB(), + StoreConfig{InitialVersion: math.MaxInt64, Pruning: types.PruneNothing}) + require.NoError(t, err) + require.Equal(t, int64(math.MaxInt64), store.Commit().Version) + require.NoError(t, err) + require.Panics(t, func() { store.Commit() }) + require.Equal(t, int64(math.MaxInt64), store.LastCommitID().Version) // version history not modified }) // setting initial version - opts := StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing} - store, err := NewStore(memdb.NewDB(), opts) + store, err := NewStore(memdb.NewDB(), + StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing, MerkleDB: memdb.NewDB()}) require.NoError(t, err) - cid := store.Commit() - require.Equal(t, int64(5), cid.Version) + require.Equal(t, int64(5), store.Commit().Version) + store.SetInitialVersion(10) + require.Equal(t, int64(10), store.Commit().Version) + + store, err = NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) + require.NoError(t, err) + store.Commit() + store.stateDB = dbVersionsFails{store.stateDB} + require.Panics(t, func() { store.LastCommitID() }) + + store, err = NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) + require.NoError(t, err) + store.Commit() + store.stateTxn = rwCrudFails{store.stateTxn} + require.Panics(t, func() { store.LastCommitID() }) } func sliceToSet(slice []uint64) map[uint64]struct{} { @@ -345,7 +441,7 @@ func TestQuery(t *testing.T) { // query subspace before anything set qres := store.Query(querySub) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, valExpSubEmpty, qres.Value) // set data @@ -354,24 +450,23 @@ func TestQuery(t *testing.T) { // set data without commit, doesn't show up qres = store.Query(query) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Nil(t, qres.Value) // commit it, but still don't see on old version cid = store.Commit() qres = store.Query(query) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Nil(t, qres.Value) // but yes on the new version query.Height = cid.Version qres = store.Query(query) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v1, qres.Value) - // and for the subspace qres = store.Query(querySub) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, valExpSub1, qres.Value) // modify @@ -380,27 +475,108 @@ func TestQuery(t *testing.T) { // query will return old values, as height is fixed qres = store.Query(query) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v1, qres.Value) // update to latest in the query and we are happy query.Height = cid.Version qres = store.Query(query) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v3, qres.Value) - query2 := abci.RequestQuery{Path: "/key", Data: k2, Height: cid.Version} + query2 := abci.RequestQuery{Path: "/key", Data: k2, Height: cid.Version} qres = store.Query(query2) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v2, qres.Value) // and for the subspace qres = store.Query(querySub) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, valExpSub2, qres.Value) // default (height 0) will show latest -1 query0 := abci.RequestQuery{Path: "/key", Data: k1} qres = store.Query(query0) - require.Equal(t, uint32(0), qres.Code) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v1, qres.Value) + + // querying an empty store will fail + short, err := NewStore(memdb.NewDB(), DefaultStoreConfig) + require.NoError(t, err) + qres = short.Query(query0) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + + // default shows latest, if latest-1 does not exist + short.Set(k1, v1) + short.Commit() + qres = short.Query(query0) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.Equal(t, v1, qres.Value) + + // query with a nil or empty key fails + badquery := abci.RequestQuery{Path: "/key", Data: []byte{}} + qres = store.Query(badquery) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + badquery.Data = nil + qres = store.Query(badquery) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + // querying an invalid height will fail + badquery = abci.RequestQuery{Path: "/key", Data: k1, Height: store.LastCommitID().Version + 1} + qres = store.Query(badquery) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + // or an invalid path + badquery = abci.RequestQuery{Path: "/badpath", Data: k1} + qres = store.Query(badquery) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + + // artificial error case + short.stateDB = dbVersionsFails{store.stateDB} + qres = short.Query(badquery) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + short.Close() + + testProve := func(name string) { + t.Logf(name) + queryProve0 := abci.RequestQuery{Path: "/key", Data: k1, Prove: true} + store.Query(queryProve0) + qres = store.Query(queryProve0) + require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.Equal(t, v1, qres.Value) + require.NotNil(t, qres.ProofOps) + } + testProve("single DB") + store.Close() + store, err = NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) + require.NoError(t, err) + store.Set(k1, v1) + store.Commit() + testProve("separate DBs") + + store.Close() +} + +type dbDeleteVersionFails struct{ dbm.DBConnection } +type dbRWCommitFails struct{ *memdb.MemDB } +type dbRWCrudFails struct{ dbm.DBConnection } +type dbSaveVersionFails struct{ *memdb.MemDB } +type dbVersionsFails struct{ dbm.DBConnection } +type rwCommitFails struct{ dbm.DBReadWriter } +type rwCrudFails struct{ dbm.DBReadWriter } + +func (dbVersionsFails) Versions() (dbm.VersionSet, error) { return nil, errors.New("dbVersionsFails") } +func (db dbRWCrudFails) ReadWriter() dbm.DBReadWriter { + return rwCrudFails{db.DBConnection.ReadWriter()} +} +func (rwCrudFails) Get([]byte) ([]byte, error) { return nil, errors.New("rwCrudFails.Get") } +func (rwCrudFails) Has([]byte) (bool, error) { return false, errors.New("rwCrudFails.Has") } +func (rwCrudFails) Set([]byte, []byte) error { return errors.New("rwCrudFails.Set") } +func (rwCrudFails) Delete([]byte) error { return errors.New("rwCrudFails.Delete") } + +func (dbSaveVersionFails) SaveVersion(uint64) error { return errors.New("dbSaveVersionFails") } +func (dbDeleteVersionFails) DeleteVersion(uint64) error { return errors.New("dbDeleteVersionFails") } +func (tx rwCommitFails) Commit() error { + tx.Discard() + return errors.New("rwCommitFails") +} +func (db dbRWCommitFails) ReadWriter() dbm.DBReadWriter { + return rwCommitFails{db.MemDB.ReadWriter()} } diff --git a/types/errors/abci.go b/types/errors/abci.go index 3f19d140ce32..d09b31b15b36 100644 --- a/types/errors/abci.go +++ b/types/errors/abci.go @@ -10,7 +10,7 @@ import ( const ( // SuccessABCICode declares an ABCI response use 0 to signal that the // processing was successful and no error is returned. - SuccessABCICode = 0 + SuccessABCICode uint32 = 0 // All unclassified errors that do not provide an ABCI code are clubbed // under an internal error code and a generic message instead of From f9130d943e6c1d35669e029bce897700831f7ecf Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 15 Oct 2021 18:01:37 +0800 Subject: [PATCH 27/36] smt store test coverage --- store/v2/smt/store.go | 22 +++++++++++++++++++++- store/v2/smt/store_test.go | 22 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/store/v2/smt/store.go b/store/v2/smt/store.go index 91ea52009bfb..f5a49adedec1 100644 --- a/store/v2/smt/store.go +++ b/store/v2/smt/store.go @@ -2,6 +2,7 @@ package smt import ( "crypto/sha256" + "errors" "time" "github.com/cosmos/cosmos-sdk/store/types" @@ -15,6 +16,11 @@ var ( _ types.BasicKVStore = (*Store)(nil) ) +var ( + errKeyEmpty error = errors.New("key is empty or nil") + errValueNil error = errors.New("value is nil") +) + // Store Implements types.KVStore and CommitKVStore. type Store struct { tree *smt.SparseMerkleTree @@ -48,6 +54,9 @@ func (s *Store) Root() []byte { return s.tree.Root() } // Get returns nil iff key doesn't exist. Panics on nil key. func (s *Store) Get(key []byte) []byte { defer telemetry.MeasureSince(time.Now(), "store", "smt", "get") + if len(key) == 0 { + panic(errKeyEmpty) + } val, err := s.tree.Get(key) if err != nil { panic(err) @@ -58,6 +67,9 @@ func (s *Store) Get(key []byte) []byte { // Has checks if a key exists. Panics on nil key. func (s *Store) Has(key []byte) bool { defer telemetry.MeasureSince(time.Now(), "store", "smt", "has") + if len(key) == 0 { + panic(errKeyEmpty) + } has, err := s.tree.Has(key) if err != nil { panic(err) @@ -68,6 +80,12 @@ func (s *Store) Has(key []byte) bool { // Set sets the key. Panics on nil key or value. func (s *Store) Set(key []byte, value []byte) { defer telemetry.MeasureSince(time.Now(), "store", "smt", "set") + if len(key) == 0 { + panic(errKeyEmpty) + } + if value == nil { + panic(errValueNil) + } _, err := s.tree.Update(key, value) if err != nil { panic(err) @@ -77,7 +95,9 @@ func (s *Store) Set(key []byte, value []byte) { // Delete deletes the key. Panics on nil key. func (s *Store) Delete(key []byte) { defer telemetry.MeasureSince(time.Now(), "store", "smt", "delete") - + if len(key) == 0 { + panic(errKeyEmpty) + } _, err := s.tree.Delete(key) if err != nil { panic(err) diff --git a/store/v2/smt/store_test.go b/store/v2/smt/store_test.go index 410fc4007e5d..1bb18fb13fd7 100644 --- a/store/v2/smt/store_test.go +++ b/store/v2/smt/store_test.go @@ -17,4 +17,26 @@ func TestGetSetHasDelete(t *testing.T) { assert.Equal(t, true, s.Has([]byte("foo"))) s.Delete([]byte("foo")) assert.Equal(t, false, s.Has([]byte("foo"))) + + assert.Panics(t, func() { s.Get(nil) }, "Get(nil key) should panic") + assert.Panics(t, func() { s.Get([]byte{}) }, "Get(empty key) should panic") + assert.Panics(t, func() { s.Has(nil) }, "Has(nil key) should panic") + assert.Panics(t, func() { s.Has([]byte{}) }, "Has(empty key) should panic") + assert.Panics(t, func() { s.Set(nil, []byte("value")) }, "Set(nil key) should panic") + assert.Panics(t, func() { s.Set([]byte{}, []byte("value")) }, "Set(empty key) should panic") + assert.Panics(t, func() { s.Set([]byte("key"), nil) }, "Set(nil value) should panic") +} + +func TestLoadStore(t *testing.T) { + nodes, values := smt.NewSimpleMap(), smt.NewSimpleMap() + s := store.NewStore(nodes, values) + + s.Set([]byte{0}, []byte{0}) + s.Set([]byte{1}, []byte{1}) + s.Delete([]byte{1}) + root := s.Root() + + s = store.LoadStore(nodes, values, root) + assert.Equal(t, []byte{0}, s.Get([]byte{0})) + assert.False(t, s.Has([]byte{1})) } From b6cd40970631205e1dd6ed0f5554d1902087acc1 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 15 Oct 2021 20:44:46 +0800 Subject: [PATCH 28/36] clean up test --- store/v2/decoupled/store_test.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 8a8f913196e2..98f73c5f076c 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -215,15 +215,8 @@ func TestCommit(t *testing.T) { require.NotEqual(t, previd.Version, id.Version) } } - for _, opts := range []StoreConfig{ - StoreConfig{Pruning: types.PruneNothing}, - StoreConfig{Pruning: types.PruneEverything}, - StoreConfig{Pruning: types.PruneNothing, MerkleDB: memdb.NewDB()}, - StoreConfig{Pruning: types.PruneEverything, MerkleDB: memdb.NewDB()}, - } { - // t.Run(fmt.Sprintf("basic with config %v", i)) - testBasic(opts) - } + testBasic(StoreConfig{Pruning: types.PruneNothing}) + testBasic(StoreConfig{Pruning: types.PruneNothing, MerkleDB: memdb.NewDB()}) testFailedCommit := func(t *testing.T, store *Store, db dbm.DBConnection) { opts := store.opts @@ -232,7 +225,6 @@ func TestCommit(t *testing.T) { } store.Set([]byte{0}, []byte{0}) - // require.Panics(t, store.Commit) require.Panics(t, func() { store.Commit() }) require.NoError(t, store.Close()) @@ -534,8 +526,8 @@ func TestQuery(t *testing.T) { require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) short.Close() - testProve := func(name string) { - t.Logf(name) + // test that proofs are generated with single and separate DBs + testProve := func() { queryProve0 := abci.RequestQuery{Path: "/key", Data: k1, Prove: true} store.Query(queryProve0) qres = store.Query(queryProve0) @@ -543,14 +535,14 @@ func TestQuery(t *testing.T) { require.Equal(t, v1, qres.Value) require.NotNil(t, qres.ProofOps) } - testProve("single DB") + testProve() store.Close() + store, err = NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) require.NoError(t, err) store.Set(k1, v1) store.Commit() - testProve("separate DBs") - + testProve() store.Close() } From 9754394b3fdfc5fc0b21b7dbc3bbfdf0900bef5b Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 18 Oct 2021 17:57:32 +0800 Subject: [PATCH 29/36] store test coverage, cleanup --- store/v2/decoupled/store.go | 7 ++- store/v2/decoupled/store_test.go | 89 +++++++++++++++++++------------- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index b4c4c0b2ad32..823603bca9f4 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -393,7 +393,7 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { height = int64(latest) } if height < 0 { - return sdkerrors.QueryResult(fmt.Errorf("height overflow: %v", latest), false) + return sdkerrors.QueryResult(fmt.Errorf("height overflow: %v", height), false) } } res.Height = height @@ -414,7 +414,7 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { contents := prefix.NewPrefixReader(dbr, dataPrefix) res.Value, err = contents.Get(res.Key) if err != nil { - return sdkerrors.QueryResult(sdkerrors.ErrKeyNotFound, false) + return sdkerrors.QueryResult(err, false) } if !req.Prove { break @@ -430,6 +430,9 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { } root, err := dbr.Get(merkleRootKey) if err != nil { + return sdkerrors.QueryResult(err, false) + } + if root == nil { return sdkerrors.QueryResult(errors.New("Merkle root hash not found"), false) } merkleStore := loadSMT(dbm.ReaderAsReadWriter(merkleView), root) diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 98f73c5f076c..13826d78cf02 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -111,17 +111,19 @@ func TestConstructors(t *testing.T) { require.Error(t, err) merkledb.Close() - // can't load existing store when we can't access the last commit hash + // can't load existing store when we can't access the latest Merkle root hash store, err = NewStore(db, DefaultStoreConfig) require.NoError(t, err) store.Commit() - + require.NoError(t, store.Close()) + // because root is misssing w = db.Writer() w.Delete(merkleRootKey) w.Commit() + db.SaveNextVersion() store, err = NewStore(db, DefaultStoreConfig) require.Error(t, err) - + // or, because of an error store, err = NewStore(dbRWCrudFails{db}, DefaultStoreConfig) require.Error(t, err) } @@ -269,33 +271,32 @@ func TestCommit(t *testing.T) { testFailedCommit(t, store, nil) }) - t.Run("stateDB.DeleteVersion error triggers failure", func(t *testing.T) { - store, err := NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) - require.NoError(t, err) - store.merkleTxn = rwCommitFails{store.merkleTxn} - store.stateDB = dbDeleteVersionFails{store.stateDB} - require.Panics(t, func() { store.Commit() }) - }) - t.Run("stateDB.Versions error triggers failure", func(t *testing.T) { + t.Run("recover after stateDB.Versions error triggers failure", func(t *testing.T) { db := memdb.NewDB() store, err := NewStore(db, DefaultStoreConfig) require.NoError(t, err) store.stateDB = dbVersionsFails{store.stateDB} testFailedCommit(t, store, db) }) - t.Run("stateTxn.Set error triggers failure", func(t *testing.T) { + t.Run("recover after stateTxn.Set error triggers failure", func(t *testing.T) { store, err := NewStore(memdb.NewDB(), DefaultStoreConfig) require.NoError(t, err) store.stateTxn = rwCrudFails{store.stateTxn} testFailedCommit(t, store, nil) }) + t.Run("stateDB.DeleteVersion error triggers failure", func(t *testing.T) { + store, err := NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) + require.NoError(t, err) + store.merkleTxn = rwCommitFails{store.merkleTxn} + store.stateDB = dbDeleteVersionFails{store.stateDB} + require.Panics(t, func() { store.Commit() }) + }) t.Run("height overflow triggers failure", func(t *testing.T) { store, err := NewStore(memdb.NewDB(), StoreConfig{InitialVersion: math.MaxInt64, Pruning: types.PruneNothing}) require.NoError(t, err) require.Equal(t, int64(math.MaxInt64), store.Commit().Version) - require.NoError(t, err) require.Panics(t, func() { store.Commit() }) require.Equal(t, int64(math.MaxInt64), store.LastCommitID().Version) // version history not modified }) @@ -342,8 +343,8 @@ func TestPruning(t *testing.T) { } for tci, tc := range testCases { - db := memdb.NewDB() - store, err := NewStore(db, StoreConfig{Pruning: tc.PruningOptions}) + dbs := []dbm.DBConnection{memdb.NewDB(), memdb.NewDB()} + store, err := NewStore(dbs[0], StoreConfig{Pruning: tc.PruningOptions, MerkleDB: dbs[1]}) require.NoError(t, err) for i := byte(1); i <= 10; i++ { @@ -353,12 +354,14 @@ func TestPruning(t *testing.T) { require.Equal(t, latest, uint64(cid.Version)) } - versions, err := db.Versions() - require.NoError(t, err) - kept := sliceToSet(tc.kept) - for v := uint64(1); v <= 10; v++ { - _, has := kept[v] - require.Equal(t, has, versions.Exists(v), "Version = %v; tc #%d", v, tci) + for _, db := range dbs { + versions, err := db.Versions() + require.NoError(t, err) + kept := sliceToSet(tc.kept) + for v := uint64(1); v <= 10; v++ { + _, has := kept[v] + require.Equal(t, has, versions.Exists(v), "Version = %v; tc #%d", v, tci) + } } } @@ -492,17 +495,30 @@ func TestQuery(t *testing.T) { require.Equal(t, v1, qres.Value) // querying an empty store will fail - short, err := NewStore(memdb.NewDB(), DefaultStoreConfig) + store2, err := NewStore(memdb.NewDB(), DefaultStoreConfig) require.NoError(t, err) - qres = short.Query(query0) + qres = store2.Query(query0) require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) // default shows latest, if latest-1 does not exist - short.Set(k1, v1) - short.Commit() - qres = short.Query(query0) + store2.Set(k1, v1) + store2.Commit() + qres = store2.Query(query0) require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) require.Equal(t, v1, qres.Value) + store2.Close() + + // artificial error cases for coverage (should never happen with defined usage) + // ensure that height overflow triggers an error + require.NoError(t, err) + store2.stateDB = dbVersionsIs{store2.stateDB, dbm.NewVersionManager([]uint64{uint64(math.MaxInt64) + 1})} + qres = store2.Query(query0) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + // failure to access versions triggers an error + store2.stateDB = dbVersionsFails{store.stateDB} + qres = store2.Query(query0) + require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + store2.Close() // query with a nil or empty key fails badquery := abci.RequestQuery{Path: "/key", Data: []byte{}} @@ -520,12 +536,6 @@ func TestQuery(t *testing.T) { qres = store.Query(badquery) require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) - // artificial error case - short.stateDB = dbVersionsFails{store.stateDB} - qres = short.Query(badquery) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) - short.Close() - // test that proofs are generated with single and separate DBs testProve := func() { queryProve0 := abci.RequestQuery{Path: "/key", Data: k1, Prove: true} @@ -550,19 +560,19 @@ type dbDeleteVersionFails struct{ dbm.DBConnection } type dbRWCommitFails struct{ *memdb.MemDB } type dbRWCrudFails struct{ dbm.DBConnection } type dbSaveVersionFails struct{ *memdb.MemDB } +type dbVersionsIs struct { + dbm.DBConnection + vset dbm.VersionSet +} type dbVersionsFails struct{ dbm.DBConnection } type rwCommitFails struct{ dbm.DBReadWriter } type rwCrudFails struct{ dbm.DBReadWriter } func (dbVersionsFails) Versions() (dbm.VersionSet, error) { return nil, errors.New("dbVersionsFails") } +func (db dbVersionsIs) Versions() (dbm.VersionSet, error) { return db.vset, nil } func (db dbRWCrudFails) ReadWriter() dbm.DBReadWriter { return rwCrudFails{db.DBConnection.ReadWriter()} } -func (rwCrudFails) Get([]byte) ([]byte, error) { return nil, errors.New("rwCrudFails.Get") } -func (rwCrudFails) Has([]byte) (bool, error) { return false, errors.New("rwCrudFails.Has") } -func (rwCrudFails) Set([]byte, []byte) error { return errors.New("rwCrudFails.Set") } -func (rwCrudFails) Delete([]byte) error { return errors.New("rwCrudFails.Delete") } - func (dbSaveVersionFails) SaveVersion(uint64) error { return errors.New("dbSaveVersionFails") } func (dbDeleteVersionFails) DeleteVersion(uint64) error { return errors.New("dbDeleteVersionFails") } func (tx rwCommitFails) Commit() error { @@ -572,3 +582,8 @@ func (tx rwCommitFails) Commit() error { func (db dbRWCommitFails) ReadWriter() dbm.DBReadWriter { return rwCommitFails{db.MemDB.ReadWriter()} } + +func (rwCrudFails) Get([]byte) ([]byte, error) { return nil, errors.New("rwCrudFails.Get") } +func (rwCrudFails) Has([]byte) (bool, error) { return false, errors.New("rwCrudFails.Has") } +func (rwCrudFails) Set([]byte, []byte) error { return errors.New("rwCrudFails.Set") } +func (rwCrudFails) Delete([]byte) error { return errors.New("rwCrudFails.Delete") } From 3a093ab70ca6cb93bec619e8c59871635b639f8d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 18 Oct 2021 21:45:26 +0800 Subject: [PATCH 30/36] unneeded method --- store/v2/decoupled/store.go | 1 - store/v2/decoupled/store_test.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 823603bca9f4..d521bc8b8b49 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -363,7 +363,6 @@ func (s *Store) LastCommitID() types.CommitID { func (s *Store) GetPruning() types.PruningOptions { return s.opts.Pruning } func (s *Store) SetPruning(po types.PruningOptions) { s.opts.Pruning = po } -func (s *Store) SetInitialVersion(version int64) { s.opts.InitialVersion = uint64(version) } // Query implements ABCI interface, allows queries. // diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index 13826d78cf02..e5c4abf3b42e 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -306,8 +306,6 @@ func TestCommit(t *testing.T) { StoreConfig{InitialVersion: 5, Pruning: types.PruneNothing, MerkleDB: memdb.NewDB()}) require.NoError(t, err) require.Equal(t, int64(5), store.Commit().Version) - store.SetInitialVersion(10) - require.Equal(t, int64(10), store.Commit().Version) store, err = NewStore(memdb.NewDB(), StoreConfig{MerkleDB: memdb.NewDB()}) require.NoError(t, err) From fe26dbb081fc51cdab18dbd80e4a66003f05ae10 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 14:44:27 +0800 Subject: [PATCH 31/36] test cleanup --- store/v2/decoupled/store_test.go | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/store/v2/decoupled/store_test.go b/store/v2/decoupled/store_test.go index e5c4abf3b42e..6fae83293c74 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/decoupled/store_test.go @@ -12,7 +12,6 @@ import ( dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/db/memdb" "github.com/cosmos/cosmos-sdk/store/types" - sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/types/kv" ) @@ -434,7 +433,7 @@ func TestQuery(t *testing.T) { // query subspace before anything set qres := store.Query(querySub) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, valExpSubEmpty, qres.Value) // set data @@ -443,23 +442,23 @@ func TestQuery(t *testing.T) { // set data without commit, doesn't show up qres = store.Query(query) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Nil(t, qres.Value) // commit it, but still don't see on old version cid = store.Commit() qres = store.Query(query) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Nil(t, qres.Value) // but yes on the new version query.Height = cid.Version qres = store.Query(query) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v1, qres.Value) // and for the subspace qres = store.Query(querySub) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, valExpSub1, qres.Value) // modify @@ -468,41 +467,41 @@ func TestQuery(t *testing.T) { // query will return old values, as height is fixed qres = store.Query(query) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v1, qres.Value) // update to latest in the query and we are happy query.Height = cid.Version qres = store.Query(query) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v3, qres.Value) query2 := abci.RequestQuery{Path: "/key", Data: k2, Height: cid.Version} qres = store.Query(query2) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v2, qres.Value) // and for the subspace qres = store.Query(querySub) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, valExpSub2, qres.Value) // default (height 0) will show latest -1 query0 := abci.RequestQuery{Path: "/key", Data: k1} qres = store.Query(query0) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v1, qres.Value) // querying an empty store will fail store2, err := NewStore(memdb.NewDB(), DefaultStoreConfig) require.NoError(t, err) qres = store2.Query(query0) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) // default shows latest, if latest-1 does not exist store2.Set(k1, v1) store2.Commit() qres = store2.Query(query0) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v1, qres.Value) store2.Close() @@ -511,35 +510,35 @@ func TestQuery(t *testing.T) { require.NoError(t, err) store2.stateDB = dbVersionsIs{store2.stateDB, dbm.NewVersionManager([]uint64{uint64(math.MaxInt64) + 1})} qres = store2.Query(query0) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) // failure to access versions triggers an error store2.stateDB = dbVersionsFails{store.stateDB} qres = store2.Query(query0) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) store2.Close() // query with a nil or empty key fails badquery := abci.RequestQuery{Path: "/key", Data: []byte{}} qres = store.Query(badquery) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) badquery.Data = nil qres = store.Query(badquery) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) // querying an invalid height will fail badquery = abci.RequestQuery{Path: "/key", Data: k1, Height: store.LastCommitID().Version + 1} qres = store.Query(badquery) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) // or an invalid path badquery = abci.RequestQuery{Path: "/badpath", Data: k1} qres = store.Query(badquery) - require.NotEqual(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsErr()) // test that proofs are generated with single and separate DBs testProve := func() { queryProve0 := abci.RequestQuery{Path: "/key", Data: k1, Prove: true} store.Query(queryProve0) qres = store.Query(queryProve0) - require.Equal(t, sdkerrors.SuccessABCICode, qres.Code) + require.True(t, qres.IsOK()) require.Equal(t, v1, qres.Value) require.NotNil(t, qres.ProofOps) } From c956d9c5900a93ceef9b992c45492997abce57c2 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 14:47:32 +0800 Subject: [PATCH 32/36] remove telemetry --- store/v2/decoupled/store.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index d521bc8b8b49..2aa5ac4ad9ed 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -7,7 +7,6 @@ import ( "io" "math" "sync" - "time" dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/db/prefix" @@ -19,7 +18,6 @@ import ( "github.com/cosmos/cosmos-sdk/store/tracekv" "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/store/v2/smt" - "github.com/cosmos/cosmos-sdk/telemetry" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/types/kv" ) @@ -152,7 +150,6 @@ func (s *Store) Close() error { // Get implements KVStore. func (s *Store) Get(key []byte) []byte { - defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "get") s.mtx.RLock() defer s.mtx.RUnlock() @@ -165,7 +162,6 @@ func (s *Store) Get(key []byte) []byte { // Has implements KVStore. func (s *Store) Has(key []byte) bool { - defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "has") s.mtx.RLock() defer s.mtx.RUnlock() @@ -178,7 +174,6 @@ func (s *Store) Has(key []byte) bool { // Set implements KVStore. func (s *Store) Set(key []byte, value []byte) { - defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "set") khash := sha256.Sum256(key) s.mtx.Lock() defer s.mtx.Unlock() @@ -196,7 +191,6 @@ func (s *Store) Set(key []byte, value []byte) { // Delete implements KVStore. func (s *Store) Delete(key []byte) { - defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "delete") khash := sha256.Sum256(key) s.mtx.Lock() defer s.mtx.Unlock() @@ -372,8 +366,6 @@ func (s *Store) SetPruning(po types.PruningOptions) { s.opts.Pruning = po } // if you care to have the latest data to see a tx results, you must // explicitly set the height you want to see func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { - defer telemetry.MeasureSince(time.Now(), "store", "decoupled", "query") - if len(req.Data) == 0 { return sdkerrors.QueryResult(sdkerrors.Wrap(sdkerrors.ErrTxDecode, "query cannot be zero length"), false) } From 3d740f235d410f5533b281c0ff606103ce3f5e3f Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 14:48:13 +0800 Subject: [PATCH 33/36] remove telemetry --- store/v2/smt/store.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/store/v2/smt/store.go b/store/v2/smt/store.go index f5a49adedec1..1873f06b078d 100644 --- a/store/v2/smt/store.go +++ b/store/v2/smt/store.go @@ -3,10 +3,8 @@ package smt import ( "crypto/sha256" "errors" - "time" "github.com/cosmos/cosmos-sdk/store/types" - "github.com/cosmos/cosmos-sdk/telemetry" tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" "github.com/lazyledger/smt" @@ -53,7 +51,6 @@ func (s *Store) Root() []byte { return s.tree.Root() } // Get returns nil iff key doesn't exist. Panics on nil key. func (s *Store) Get(key []byte) []byte { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "get") if len(key) == 0 { panic(errKeyEmpty) } @@ -66,7 +63,6 @@ func (s *Store) Get(key []byte) []byte { // Has checks if a key exists. Panics on nil key. func (s *Store) Has(key []byte) bool { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "has") if len(key) == 0 { panic(errKeyEmpty) } @@ -79,7 +75,6 @@ func (s *Store) Has(key []byte) bool { // Set sets the key. Panics on nil key or value. func (s *Store) Set(key []byte, value []byte) { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "set") if len(key) == 0 { panic(errKeyEmpty) } @@ -94,7 +89,6 @@ func (s *Store) Set(key []byte, value []byte) { // Delete deletes the key. Panics on nil key. func (s *Store) Delete(key []byte) { - defer telemetry.MeasureSince(time.Now(), "store", "smt", "delete") if len(key) == 0 { panic(errKeyEmpty) } From 15c5118c80a485059e0a13f80a0a34459c46a31c Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 19:03:03 +0800 Subject: [PATCH 34/36] PR revisions --- db/dbtest/util.go | 10 ++++------ db/go.mod | 4 ++-- db/internal/util.go | 2 +- db/prefix/prefix.go | 2 +- db/prefix/prefix_iterator.go | 12 +++++------- store/v2/decoupled/store.go | 12 ++++++------ store/v2/smt/proof.go | 2 +- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/db/dbtest/util.go b/db/dbtest/util.go index f65ac87e9fec..2ddb91929382 100644 --- a/db/dbtest/util.go +++ b/db/dbtest/util.go @@ -21,12 +21,10 @@ func AssertDomain(t *testing.T, itr dbm.Iterator, start, end []byte) { assert.Equal(t, end, de, "checkDomain domain end incorrect") } -func AssertItem(t *testing.T, itr dbm.Iterator, key []byte, value []byte) { +func AssertItem(t *testing.T, itr dbm.Iterator, key, value []byte) { t.Helper() - v := itr.Value() - k := itr.Key() - assert.Exactly(t, key, k) - assert.Exactly(t, value, v) + assert.Exactly(t, itr.Key(), k) + assert.Exactly(t, itr.Value(), v) } func AssertInvalid(t *testing.T, itr dbm.Iterator) { @@ -41,7 +39,7 @@ func AssertKeyPanics(t *testing.T, itr dbm.Iterator) { assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") } -func AssertValue(t *testing.T, db dbm.DBReader, key []byte, valueWanted []byte) { +func AssertValue(t *testing.T, db dbm.DBReader, key, valueWanted []byte) { t.Helper() valueGot, err := db.Get(key) assert.NoError(t, err) diff --git a/db/go.mod b/db/go.mod index 5f35591230fe..0577e38ffe87 100644 --- a/db/go.mod +++ b/db/go.mod @@ -1,7 +1,7 @@ -module github.com/cosmos/cosmos-sdk/db - go 1.17 +module github.com/cosmos/cosmos-sdk/db + require ( github.com/dgraph-io/badger/v3 v3.2103.1 github.com/google/btree v1.0.0 diff --git a/db/internal/util.go b/db/internal/util.go index 310442c0212d..b33f8ac67733 100644 --- a/db/internal/util.go +++ b/db/internal/util.go @@ -18,7 +18,7 @@ func ValidateKv(key, value []byte) error { func CombineErrors(ret error, also error, desc string) error { if also != nil { if ret != nil { - ret = fmt.Errorf("%w; %v: %v", ret, desc, also) + ret = fmt.Errorf("%w; %s: %v", ret, desc, also) } else { ret = also } diff --git a/db/prefix/prefix.go b/db/prefix/prefix.go index 26ac2741d778..6116c79203df 100644 --- a/db/prefix/prefix.go +++ b/db/prefix/prefix.go @@ -32,7 +32,7 @@ func NewPrefixReadWriter(db dbm.DBReadWriter, prefix []byte) prefixRW { } } -func prefixed(prefix []byte, key []byte) []byte { +func prefixed(prefix, key []byte) []byte { return append(prefix, key...) } diff --git a/db/prefix/prefix_iterator.go b/db/prefix/prefix_iterator.go index 7a8169e11986..0b7f1c09ef36 100644 --- a/db/prefix/prefix_iterator.go +++ b/db/prefix/prefix_iterator.go @@ -11,10 +11,7 @@ import ( // restricted by prefix. func IteratePrefix(db dbm.DBReader, prefix []byte) (dbm.Iterator, error) { var start, end []byte - if len(prefix) == 0 { - start = nil - end = nil - } else { + if len(prefix) != 0 { start = prefix end = cpIncr(prefix) } @@ -46,7 +43,7 @@ func newPrefixIterator(prefix, start, end []byte, source dbm.Iterator) *prefixDB } // Domain implements Iterator. -func (itr *prefixDBIterator) Domain() (start []byte, end []byte) { +func (itr *prefixDBIterator) Domain() (start, end []byte) { return itr.start, itr.end } @@ -70,12 +67,13 @@ func (itr *prefixDBIterator) Next() bool { if !itr.source.Next() { return false } - if !bytes.HasPrefix(itr.source.Key(), itr.prefix) { + key := itr.source.Key() + if !bytes.HasPrefix(key, itr.prefix) { return false } // Empty keys are not allowed, so if a key exists in the database that exactly matches the // prefix we need to skip it. - if bytes.Equal(itr.source.Key(), itr.prefix) { + if bytes.Equal(key, itr.prefix) { return itr.Next() } return true diff --git a/store/v2/decoupled/store.go b/store/v2/decoupled/store.go index 2aa5ac4ad9ed..c57f635e316a 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/decoupled/store.go @@ -173,8 +173,7 @@ func (s *Store) Has(key []byte) bool { } // Set implements KVStore. -func (s *Store) Set(key []byte, value []byte) { - khash := sha256.Sum256(key) +func (s *Store) Set(key, value []byte) { s.mtx.Lock() defer s.mtx.Unlock() @@ -183,6 +182,7 @@ func (s *Store) Set(key []byte, value []byte) { panic(err) } s.merkleStore.Set(key, value) + khash := sha256.Sum256(key) err = s.indexTxn.Set(khash[:], key) if err != nil { panic(err) @@ -383,9 +383,9 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { } else { height = int64(latest) } - if height < 0 { - return sdkerrors.QueryResult(fmt.Errorf("height overflow: %v", height), false) - } + } + if height < 0 { + return sdkerrors.QueryResult(fmt.Errorf("height overflow: %v", height), false) } res.Height = height @@ -396,7 +396,7 @@ func (s *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { dbr, err := s.stateDB.ReaderAt(uint64(height)) if err != nil { - if err == dbm.ErrVersionDoesNotExist { + if errors.Is(err, dbm.ErrVersionDoesNotExist) { err = sdkerrors.ErrInvalidHeight } return sdkerrors.QueryResult(err, false) diff --git a/store/v2/smt/proof.go b/store/v2/smt/proof.go index e606b0f649b3..f013d1b2b995 100644 --- a/store/v2/smt/proof.go +++ b/store/v2/smt/proof.go @@ -30,7 +30,7 @@ type ProofOp struct { Proof smt.SparseMerkleProof } -var _ merkle.ProofOperator = &ProofOp{} +var _ merkle.ProofOperator = (*ProofOp)(nil) // NewProofOp returns a ProofOp for a SparseMerkleProof. func NewProofOp(root, key []byte, hasher HasherType, proof smt.SparseMerkleProof) *ProofOp { From 5fc15d5aceae56a12613e371337b39207403c9a0 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 19:25:01 +0800 Subject: [PATCH 35/36] rename decoupled -> flat --- docs/core/store.md | 10 +++++----- store/v2/{decoupled => flat}/store.go | 2 +- store/v2/{decoupled => flat}/store_test.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename store/v2/{decoupled => flat}/store.go (99%) rename store/v2/{decoupled => flat}/store_test.go (99%) diff --git a/docs/core/store.md b/docs/core/store.md index a3807bf3ee4a..6f74afcf1fd7 100644 --- a/docs/core/store.md +++ b/docs/core/store.md @@ -228,19 +228,19 @@ The SDK is in the process of transitioning to use the types listed here as the d ### `BasicKVStore` interface -An interface providing only the basic CRUD functionality (`Get`, `Set`, `Has`, and `Delete` methods), without iteration or caching. This is used to partially expose components of a larger store, such as a `decoupled.Store`. +An interface providing only the basic CRUD functionality (`Get`, `Set`, `Has`, and `Delete` methods), without iteration or caching. This is used to partially expose components of a larger store, such as a `flat.Store`. -### Decoupled Store +### Flat Store -`decoupled.Store` is the new default persistent store, which internally decouples the concerns of state storage and commitment scheme. Values are stored directly in the backing key-value database (the "storage" bucket), while the value's hash is mapped in a separate store which is able to generate a cryptographic commitment (the "state commitment" bucket, implmented with `smt.Store`). +`flat.Store` is the new default persistent store, which internally decouples the concerns of state storage and commitment scheme. Values are stored directly in the backing key-value database (the "storage" bucket), while the value's hash is mapped in a separate store which is able to generate a cryptographic commitment (the "state commitment" bucket, implmented with `smt.Store`). This can optionally be constructed to use different backend databases for each bucket. - + ### SMT Store -A `BasicKVStore` which is used to partially expose functions of an underlying store (for instance, to allow access to the commitment store in `decoupled.Store`). +A `BasicKVStore` which is used to partially expose functions of an underlying store (for instance, to allow access to the commitment store in `flat.Store`). ## Next {hide} diff --git a/store/v2/decoupled/store.go b/store/v2/flat/store.go similarity index 99% rename from store/v2/decoupled/store.go rename to store/v2/flat/store.go index c57f635e316a..2ea17f16cf7a 100644 --- a/store/v2/decoupled/store.go +++ b/store/v2/flat/store.go @@ -1,4 +1,4 @@ -package decoupled +package flat import ( "crypto/sha256" diff --git a/store/v2/decoupled/store_test.go b/store/v2/flat/store_test.go similarity index 99% rename from store/v2/decoupled/store_test.go rename to store/v2/flat/store_test.go index 6fae83293c74..c3f559b31c2f 100644 --- a/store/v2/decoupled/store_test.go +++ b/store/v2/flat/store_test.go @@ -1,4 +1,4 @@ -package decoupled +package flat import ( "errors" From ec86707b9ed1a0806a0fa155eef9635053e806f8 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 19 Oct 2021 19:29:51 +0800 Subject: [PATCH 36/36] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f059ebd001a..4e7910ac00b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -199,6 +199,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [\#9848](https://github.com/cosmos/cosmos-sdk/pull/9848) ADR-040: Implement BadgerDB backend * [\#9851](https://github.com/cosmos/cosmos-sdk/pull/9851) ADR-040: Implement RocksDB backend * [\#10308](https://github.com/cosmos/cosmos-sdk/pull/10308) ADR-040: Implement DBConnection.Revert +* [\#9892](https://github.com/cosmos/cosmos-sdk/pull/9892) ADR-040: KV Store with decoupled storage and state commitment ### Client Breaking Changes