From 14ad9eb55de4b02303f46acf5f3f016db3be7e83 Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Tue, 7 Sep 2021 16:34:30 -0400 Subject: [PATCH 01/10] add code to parse limit param --- lib/runtime/wasmer/imports.go | 17 +++++++++++++++-- lib/runtime/wasmer/imports_test.go | 21 +++++++++++++++++++++ lib/runtime/wasmer/test_helpers.go | 2 ++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 7b0573f8ec..08bb9132aa 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -59,7 +59,7 @@ package wasmer // extern int64_t ext_default_child_storage_root_version_1(void *context, int64_t a); // extern void ext_default_child_storage_set_version_1(void *context, int64_t a, int64_t b, int64_t c); // extern void ext_default_child_storage_storage_kill_version_1(void *context, int64_t a); -// extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int64_t b); +// extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int32_t b); // extern int64_t ext_default_child_storage_storage_kill_version_3(void *context, int64_t a, int64_t b); // extern void ext_default_child_storage_clear_prefix_version_1(void *context, int64_t a, int64_t b); // extern int32_t ext_default_child_storage_exists_version_1(void *context, int64_t a, int64_t b); @@ -1120,7 +1120,7 @@ func ext_default_child_storage_storage_kill_version_1(context unsafe.Pointer, ch } //export ext_default_child_storage_storage_kill_version_2 -func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan, _ C.int64_t) C.int32_t { +func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan C.int64_t, lim C.int32_t) C.int32_t { logger.Debug("[ext_default_child_storage_storage_kill_version_2] executing...") logger.Warn("[ext_default_child_storage_storage_kill_version_2] somewhat unimplemented") // TODO: need to use `limit` parameter @@ -1132,6 +1132,19 @@ func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, ch childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) storage.DeleteChild(childStorageKey) + //todo (ed) confirm this is correct way to read limit param + limitBuf := make([]byte, 4) + binary.LittleEndian.PutUint32(limitBuf, uint32(lim)) + buf := &bytes.Buffer{} + buf.Write(limitBuf) + + limit, err := optional.NewBytes(false, nil).Decode(buf) + if err != nil { + logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot generate limit", "error", err) + return 0 + } + fmt.Printf("limit %v\n", limit) + // note: this function always returns `KillStorageResult::AllRemoved`, which is 0 return 0 } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index aba4174383..fbcf6de02d 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -1050,6 +1050,27 @@ func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { require.Nil(t, child) } +func Test_ext_default_child_storage_storage_kill_version_2(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + require.NoError(t, err) + + // Confirm if value is set + child, err := inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.NotNil(t, child) + + encChildKey, err := scale.Encode(testChildKey) + require.NoError(t, err) + + _, err = inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", encChildKey) + require.NoError(t, err) + + child, _ = inst.ctx.Storage.GetChild(testChildKey) + require.Nil(t, child) +} + func Test_ext_storage_append_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) diff --git a/lib/runtime/wasmer/test_helpers.go b/lib/runtime/wasmer/test_helpers.go index f9e9d91690..4bab111e4d 100644 --- a/lib/runtime/wasmer/test_helpers.go +++ b/lib/runtime/wasmer/test_helpers.go @@ -17,6 +17,7 @@ package wasmer import ( + "fmt" "path/filepath" "testing" @@ -58,6 +59,7 @@ func NewTestInstanceWithRole(t *testing.T, targetRuntime string, role byte) *Ins func setupConfig(t *testing.T, targetRuntime string, tt *trie.Trie, lvl log.Lvl, role byte) (string, *Config) { testRuntimeFilePath, testRuntimeURL := runtime.GetRuntimeVars(targetRuntime) +fmt.Printf("RT PATH %v\n", testRuntimeFilePath) _, err := runtime.GetRuntimeBlob(testRuntimeFilePath, testRuntimeURL) require.Nil(t, err, "Fail: could not get runtime", "targetRuntime", targetRuntime) From d98c6b9a217d56fc85ae43ac50f70486f241036a Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Mon, 20 Sep 2021 17:43:04 -0400 Subject: [PATCH 02/10] add optional encoding for limit --- lib/runtime/wasmer/imports.go | 7 +++++-- lib/runtime/wasmer/imports_test.go | 12 +++++++++++- lib/runtime/wasmer/test_helpers.go | 2 -- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 08bb9132aa..07f10d8312 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1132,19 +1132,22 @@ func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, ch childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) storage.DeleteChild(childStorageKey) - //todo (ed) confirm this is correct way to read limit param + fmt.Printf("lim %v\n", lim) limitBuf := make([]byte, 4) binary.LittleEndian.PutUint32(limitBuf, uint32(lim)) buf := &bytes.Buffer{} buf.Write(limitBuf) - limit, err := optional.NewBytes(false, nil).Decode(buf) + limit, err := optional.NewBytes(true, nil).Decode(buf) if err != nil { logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot generate limit", "error", err) return 0 } fmt.Printf("limit %v\n", limit) + limitUint := binary.LittleEndian.Uint32(limit.Value()) + fmt.Printf(" limit uint %v\n", limitUint) + // note: this function always returns `KillStorageResult::AllRemoved`, which is 0 return 0 } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index fbcf6de02d..b7ffec1f97 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -18,6 +18,8 @@ package wasmer import ( "bytes" + "encoding/binary" + "fmt" "os" "sort" "testing" @@ -1064,7 +1066,15 @@ func Test_ext_default_child_storage_storage_kill_version_2(t *testing.T) { encChildKey, err := scale.Encode(testChildKey) require.NoError(t, err) - _, err = inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", encChildKey) + testLimit := uint32(2) + testLimitBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(testLimitBytes, testLimit) + + optLim, err := optional.NewBytes(true, testLimitBytes).Encode() + require.NoError(t, err) + fmt.Printf("optLimit %v\n", optLim) + + _, err = inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLim...)) require.NoError(t, err) child, _ = inst.ctx.Storage.GetChild(testChildKey) diff --git a/lib/runtime/wasmer/test_helpers.go b/lib/runtime/wasmer/test_helpers.go index 4bab111e4d..f9e9d91690 100644 --- a/lib/runtime/wasmer/test_helpers.go +++ b/lib/runtime/wasmer/test_helpers.go @@ -17,7 +17,6 @@ package wasmer import ( - "fmt" "path/filepath" "testing" @@ -59,7 +58,6 @@ func NewTestInstanceWithRole(t *testing.T, targetRuntime string, role byte) *Ins func setupConfig(t *testing.T, targetRuntime string, tt *trie.Trie, lvl log.Lvl, role byte) (string, *Config) { testRuntimeFilePath, testRuntimeURL := runtime.GetRuntimeVars(targetRuntime) -fmt.Printf("RT PATH %v\n", testRuntimeFilePath) _, err := runtime.GetRuntimeBlob(testRuntimeFilePath, testRuntimeURL) require.Nil(t, err, "Fail: could not get runtime", "targetRuntime", targetRuntime) From 630cf49b61f06c125fbb76815191e43b50d3504f Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Tue, 21 Sep 2021 16:29:54 -0400 Subject: [PATCH 03/10] implement functionity for ext function --- lib/runtime/constants.go | 6 ++- lib/runtime/wasmer/imports.go | 37 ++++++++++---- lib/runtime/wasmer/imports_test.go | 78 +++++++++++++++++++++++++++--- 3 files changed, 105 insertions(+), 16 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index a691a7fc67..1a108b9e64 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -42,7 +42,11 @@ const ( // v0.8 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" + //HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" + // todo (ed) use above URL once ed/add_rtm_ext_default_child_storage_storage_kill_version_2 has been merged to master + HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/ed/add_rtm_ext_default_child_storage_storage_kill_version_2/test/hostapi_runtime.compact.wasm?raw=true" + + // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 07f10d8312..816e25457f 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -113,6 +113,7 @@ import ( "math/big" "math/rand" "reflect" + "sort" "unsafe" "github.com/ChainSafe/gossamer/dot/types" @@ -1122,17 +1123,12 @@ func ext_default_child_storage_storage_kill_version_1(context unsafe.Pointer, ch //export ext_default_child_storage_storage_kill_version_2 func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan C.int64_t, lim C.int32_t) C.int32_t { logger.Debug("[ext_default_child_storage_storage_kill_version_2] executing...") - logger.Warn("[ext_default_child_storage_storage_kill_version_2] somewhat unimplemented") - // TODO: need to use `limit` parameter instanceContext := wasm.IntoInstanceContext(context) ctx := instanceContext.Data().(*runtime.Context) storage := ctx.Storage - childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - storage.DeleteChild(childStorageKey) - fmt.Printf("lim %v\n", lim) limitBuf := make([]byte, 4) binary.LittleEndian.PutUint32(limitBuf, uint32(lim)) buf := &bytes.Buffer{} @@ -1143,12 +1139,35 @@ func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, ch logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot generate limit", "error", err) return 0 } - fmt.Printf("limit %v\n", limit) + limitUint := uint32(0) + if limit.Exists() { + limitUint = binary.LittleEndian.Uint32(limit.Value()) + } + + tr, err := storage.GetChild(childStorageKey) + if err != nil { + logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot get child storage", "error", err) + } - limitUint := binary.LittleEndian.Uint32(limit.Value()) - fmt.Printf(" limit uint %v\n", limitUint) + if int(limitUint) >= len(tr.Entries()) || !limit.Exists() { + storage.DeleteChild(childStorageKey) + return 0 + } + + keys := make([]string, 0, len(tr.Entries())) + for k := range tr.Entries() { + keys = append(keys, k) + } + sort.Strings(keys) + deleted := uint32(0) + for _, k := range keys { + tr.Delete([]byte(k)) + deleted++ + if deleted == limitUint { + return 1 + } + } - // note: this function always returns `KillStorageResult::AllRemoved`, which is 0 return 0 } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index b7ffec1f97..cac03a0d57 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -19,7 +19,6 @@ package wasmer import ( "bytes" "encoding/binary" - "fmt" "os" "sort" "testing" @@ -1052,10 +1051,13 @@ func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { require.Nil(t, child) } -func Test_ext_default_child_storage_storage_kill_version_2(t *testing.T) { +func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + tr := trie.NewEmptyTrie() + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) // Confirm if value is set @@ -1070,17 +1072,81 @@ func Test_ext_default_child_storage_storage_kill_version_2(t *testing.T) { testLimitBytes := make([]byte, 4) binary.LittleEndian.PutUint32(testLimitBytes, testLimit) - optLim, err := optional.NewBytes(true, testLimitBytes).Encode() + optLimit, err := optional.NewBytes(true, testLimitBytes).Encode() require.NoError(t, err) - fmt.Printf("optLimit %v\n", optLim) - _, err = inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLim...)) + res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) require.NoError(t, err) + require.Equal(t, []byte{0, 0, 0, 0}, res) child, _ = inst.ctx.Storage.GetChild(testChildKey) require.Nil(t, child) } +func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + tr := trie.NewEmptyTrie() + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.ctx.Storage.SetChild(testChildKey, tr) + require.NoError(t, err) + + // Confirm if value is set + child, err := inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.NotNil(t, child) + + encChildKey, err := scale.Encode(testChildKey) + require.NoError(t, err) + + testLimit := uint32(1) + testLimitBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(testLimitBytes, testLimit) + + optLimit, err := optional.NewBytes(true, testLimitBytes).Encode() + require.NoError(t, err) + + res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) + require.NoError(t, err) + require.Equal(t, []byte{1, 0, 0, 0}, res) + + child, err = inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.Equal(t, 1, len(child.Entries())) +} + +func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + tr := trie.NewEmptyTrie() + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.ctx.Storage.SetChild(testChildKey, tr) + require.NoError(t, err) + + // Confirm if value is set + child, err := inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.NotNil(t, child) + + encChildKey, err := scale.Encode(testChildKey) + require.NoError(t, err) + + optLimit, err := optional.NewBytes(false, nil).Encode() + require.NoError(t, err) + + //TODO determine why this call fails with Error: Received unexpected error: Failed to call the `rtm_ext_default_child_storage_storage_kill_version_2` exported function. + // when passing limit optional as false + res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) + require.NoError(t, err) + require.Equal(t, []byte{0, 0, 0, 0}, res) + + child, err = inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.Nil(t, child) +} + func Test_ext_storage_append_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) From ec158ed68ea64ec50540bba7d30951902758a5ab Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Tue, 21 Sep 2021 16:49:14 -0400 Subject: [PATCH 04/10] lint --- lib/runtime/constants.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 1a108b9e64..466018617a 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -40,14 +40,12 @@ const ( POLKADOT_RUNTIME_URL = "https://github.com/noot/polkadot/blob/noot/v0.8.25/polkadot_runtime.wasm?raw=true" // v0.8 test API wasm - HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME = "hostapi_runtime" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" //HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" // todo (ed) use above URL once ed/add_rtm_ext_default_child_storage_storage_kill_version_2 has been merged to master HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/ed/add_rtm_ext_default_child_storage_storage_kill_version_2/test/hostapi_runtime.compact.wasm?raw=true" - - // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" DEV_RUNTIME_FP = "dev_runtime.compact.wasm" From 25eea7292ffd70b593f19d2493aabdaac38aab50 Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Mon, 27 Sep 2021 16:44:47 -0400 Subject: [PATCH 05/10] implement trie.DeleteChildLimit function --- lib/runtime/interface.go | 2 ++ lib/runtime/storage/trie.go | 39 ++++++++++++++++++++++++++++++ lib/runtime/wasmer/imports.go | 37 ++++++---------------------- lib/runtime/wasmer/imports_test.go | 8 +++--- 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/lib/runtime/interface.go b/lib/runtime/interface.go index 1448505524..399c17f67a 100644 --- a/lib/runtime/interface.go +++ b/lib/runtime/interface.go @@ -19,6 +19,7 @@ package runtime import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/common/optional" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/transaction" "github.com/ChainSafe/gossamer/lib/trie" @@ -66,6 +67,7 @@ type Storage interface { GetChildStorage(keyToChild, key []byte) ([]byte, error) Delete(key []byte) DeleteChild(keyToChild []byte) + DeleteChildLimit(keyToChild []byte, limit *optional.Bytes) (uint32, bool, error) ClearChildStorage(keyToChild, key []byte) error NextKey([]byte) []byte ClearPrefixInChild(keyToChild, prefix []byte) error diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 7878fde7a0..12927f71dc 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -18,9 +18,11 @@ package storage import ( "encoding/binary" + "sort" "sync" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/common/optional" "github.com/ChainSafe/gossamer/lib/trie" ) @@ -178,6 +180,43 @@ func (s *TrieState) DeleteChild(key []byte) { s.t.DeleteChild(key) } +// DeleteChildLimit deletes up to limit of database entries by lexicographic order, return number +// deleted, true if all delete atherwise false +func (s *TrieState) DeleteChildLimit(key []byte, limit *optional.Bytes) (uint32, bool, error) { + s.lock.Lock() + defer s.lock.Unlock() + tr, err := s.t.GetChild(key) + if err != nil { + return 0, false, err + } + qtyEntries := uint32(len(tr.Entries())) + if limit == nil || !limit.Exists() { + s.t.DeleteChild(key) + return qtyEntries, true, nil + } + limitUint := binary.LittleEndian.Uint32(limit.Value()) + + keys := make([]string, 0, len(tr.Entries())) + for k := range tr.Entries() { + keys = append(keys, k) + } + sort.Strings(keys) + deleted := uint32(0) + for _, k := range keys { + tr.Delete([]byte(k)) + deleted++ + if deleted == limitUint { + break + } + } + + if deleted == qtyEntries { + return deleted, true, nil + } + + return deleted, false, nil +} + // ClearChildStorage removes the child storage entry from the trie func (s *TrieState) ClearChildStorage(keyToChild, key []byte) error { s.lock.Lock() diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 07e16a7aaa..aa92e3b381 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -59,7 +59,7 @@ package wasmer // extern int64_t ext_default_child_storage_root_version_1(void *context, int64_t a); // extern void ext_default_child_storage_set_version_1(void *context, int64_t a, int64_t b, int64_t c); // extern void ext_default_child_storage_storage_kill_version_1(void *context, int64_t a); -// extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int32_t b); +// extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int64_t b); // extern int64_t ext_default_child_storage_storage_kill_version_3(void *context, int64_t a, int64_t b); // extern void ext_default_child_storage_clear_prefix_version_1(void *context, int64_t a, int64_t b); // extern int32_t ext_default_child_storage_exists_version_1(void *context, int64_t a, int64_t b); @@ -113,7 +113,6 @@ import ( "math/big" "math/rand" "reflect" - "sort" "unsafe" "github.com/ChainSafe/gossamer/dot/types" @@ -1121,7 +1120,7 @@ func ext_default_child_storage_storage_kill_version_1(context unsafe.Pointer, ch } //export ext_default_child_storage_storage_kill_version_2 -func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan C.int64_t, lim C.int32_t) C.int32_t { +func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan C.int64_t, lim C.int64_t) C.int32_t { logger.Debug("[ext_default_child_storage_storage_kill_version_2] executing...") instanceContext := wasm.IntoInstanceContext(context) @@ -1129,46 +1128,26 @@ func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, ch storage := ctx.Storage childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - limitBuf := make([]byte, 4) - binary.LittleEndian.PutUint32(limitBuf, uint32(lim)) + limitBytes := asMemorySlice(instanceContext, lim) buf := &bytes.Buffer{} - buf.Write(limitBuf) + buf.Write(limitBytes) limit, err := optional.NewBytes(true, nil).Decode(buf) if err != nil { logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot generate limit", "error", err) return 0 } - limitUint := uint32(0) - if limit.Exists() { - limitUint = binary.LittleEndian.Uint32(limit.Value()) - } - tr, err := storage.GetChild(childStorageKey) + _, all, err := storage.DeleteChildLimit(childStorageKey, limit) if err != nil { logger.Warn("[ext_default_child_storage_storage_kill_version_2] cannot get child storage", "error", err) } - if int(limitUint) >= len(tr.Entries()) || !limit.Exists() { - storage.DeleteChild(childStorageKey) + if all { + return 1 + } else { return 0 } - - keys := make([]string, 0, len(tr.Entries())) - for k := range tr.Entries() { - keys = append(keys, k) - } - sort.Strings(keys) - deleted := uint32(0) - for _, k := range keys { - tr.Delete([]byte(k)) - deleted++ - if deleted == limitUint { - return 1 - } - } - - return 0 } //export ext_default_child_storage_storage_kill_version_3 diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 79e7a1e0b0..e28c636288 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -1130,7 +1130,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) require.NoError(t, err) - require.Equal(t, []byte{1, 0, 0, 0}, res) + require.Equal(t, []byte{0, 0, 0, 0}, res) child, err = inst.ctx.Storage.GetChild(testChildKey) require.NoError(t, err) @@ -1157,14 +1157,12 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing optLimit, err := optional.NewBytes(false, nil).Encode() require.NoError(t, err) - //TODO determine why this call fails with Error: Received unexpected error: Failed to call the `rtm_ext_default_child_storage_storage_kill_version_2` exported function. - // when passing limit optional as false res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) require.NoError(t, err) - require.Equal(t, []byte{0, 0, 0, 0}, res) + require.Equal(t, []byte{1, 0, 0, 0}, res) child, err = inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) + require.Error(t, err) require.Nil(t, child) } From d08925efa0f328bc993912f4cc70d3fd2a447dfe Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Thu, 30 Sep 2021 15:34:13 -0400 Subject: [PATCH 06/10] address PR comments --- lib/runtime/storage/trie.go | 2 +- lib/runtime/wasmer/imports.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 12927f71dc..aa9a97cb2a 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -181,7 +181,7 @@ func (s *TrieState) DeleteChild(key []byte) { } // DeleteChildLimit deletes up to limit of database entries by lexicographic order, return number -// deleted, true if all delete atherwise false +// deleted, true if all delete otherwise false func (s *TrieState) DeleteChildLimit(key []byte, limit *optional.Bytes) (uint32, bool, error) { s.lock.Lock() defer s.lock.Unlock() diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index aa92e3b381..1866a7feec 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1145,9 +1145,9 @@ func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, ch if all { return 1 - } else { - return 0 } + + return 0 } //export ext_default_child_storage_storage_kill_version_3 From 6922869f9129679846d5b1398e1ca8ba9753977f Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Fri, 1 Oct 2021 13:51:26 -0400 Subject: [PATCH 07/10] update HOST_API_TEST_RUNTIME_URL to use master branch --- lib/runtime/constants.go | 4 +--- lib/runtime/wasmer/imports_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 466018617a..11045db618 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -42,9 +42,7 @@ const ( // v0.8 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - //HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" - // todo (ed) use above URL once ed/add_rtm_ext_default_child_storage_storage_kill_version_2 has been merged to master - HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/ed/add_rtm_ext_default_child_storage_storage_kill_version_2/test/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index e28c636288..a8cf31ff53 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -1098,10 +1098,11 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) require.NoError(t, err) - require.Equal(t, []byte{0, 0, 0, 0}, res) + require.Equal(t, []byte{1, 0, 0, 0}, res) - child, _ = inst.ctx.Storage.GetChild(testChildKey) - require.Nil(t, child) + child, err = inst.ctx.Storage.GetChild(testChildKey) + require.NoError(t, err) + require.Equal(t, 0, len(child.Entries())) } func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) { From 3a80c2a3740a94e4f0a3495fa5b4a828961dec3f Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Fri, 1 Oct 2021 14:21:15 -0400 Subject: [PATCH 08/10] lint --- lib/runtime/constants.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 11045db618..a691a7fc67 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -40,8 +40,8 @@ const ( POLKADOT_RUNTIME_URL = "https://github.com/noot/polkadot/blob/noot/v0.8.25/polkadot_runtime.wasm?raw=true" // v0.8 test API wasm - HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME = "hostapi_runtime" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" HOST_API_TEST_RUNTIME_URL = "https://github.com/noot/polkadot-spec/blob/master/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) From 4b318ff5e7b4ff37995fffaa24959496c6aaad9a Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Wed, 6 Oct 2021 10:39:37 -0400 Subject: [PATCH 09/10] combine parameters --- lib/runtime/wasmer/imports.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 3f7ae7659f..0c5a928f0d 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1118,7 +1118,7 @@ func ext_default_child_storage_storage_kill_version_1(context unsafe.Pointer, ch } //export ext_default_child_storage_storage_kill_version_2 -func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan C.int64_t, lim C.int64_t) C.int32_t { +func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, childStorageKeySpan, lim C.int64_t) C.int32_t { logger.Debug("[ext_default_child_storage_storage_kill_version_2] executing...") instanceContext := wasm.IntoInstanceContext(context) From eca7c15f17dba3b801d38ee3f6f43ccb79087029 Mon Sep 17 00:00:00 2001 From: Edward Mack Date: Thu, 7 Oct 2021 15:15:17 -0400 Subject: [PATCH 10/10] add unit test for DeleteChildLimit --- lib/runtime/storage/trie_test.go | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index c2e22ad62a..dcdb41fe3d 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -18,10 +18,12 @@ package storage import ( "bytes" + "encoding/binary" "sort" "testing" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/common/optional" "github.com/ChainSafe/gossamer/lib/trie" "github.com/stretchr/testify/require" ) @@ -197,3 +199,50 @@ func TestTrieState_RollbackStorageTransaction(t *testing.T) { val := ts.Get([]byte(testCases[0])) require.Equal(t, []byte(testCases[0]), val) } + +func TestTrieState_DeleteChildLimit(t *testing.T) { + ts := newTestTrieState(t) + child := trie.NewEmptyTrie() + + keys := []string{ + "key3", + "key1", + "key2", + } + + for i, key := range keys { + child.Put([]byte(key), []byte{byte(i)}) + } + + keyToChild := []byte("keytochild") + + err := ts.SetChild(keyToChild, child) + require.NoError(t, err) + + testLimitBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(testLimitBytes, uint32(2)) + optLimit2 := optional.NewBytes(true, testLimitBytes) + + testCases := []struct { + key []byte + limit *optional.Bytes + expectedDeleted uint32 + expectedDelAll bool + errMsg string + }{ + {key: []byte("fakekey"), limit: optLimit2, expectedDeleted: 0, expectedDelAll: false, errMsg: "child trie does not exist at key :child_storage:default:fakekey"}, + {key: []byte("keytochild"), limit: optLimit2, expectedDeleted: 2, expectedDelAll: false}, + {key: []byte("keytochild"), limit: nil, expectedDeleted: 1, expectedDelAll: true}, + } + for _, test := range testCases { + deleted, all, err := ts.DeleteChildLimit(test.key, test.limit) + if test.errMsg != "" { + require.Error(t, err) + require.EqualError(t, err, test.errMsg) + continue + } + require.NoError(t, err) + require.Equal(t, test.expectedDeleted, deleted) + require.Equal(t, test.expectedDelAll, all) + } +}