From 4855d9304e80978e8fcf3fafb9d52662bdcd09e4 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 15 Mar 2022 17:10:26 +0530 Subject: [PATCH 1/6] Index Removed storage diffs on contract destruction --- statediff/builder.go | 89 ++++++++++++++++++++-- statediff/indexer/database/dump/indexer.go | 33 ++++---- statediff/indexer/database/file/indexer.go | 33 ++++---- statediff/indexer/database/sql/indexer.go | 32 ++++---- 4 files changed, 135 insertions(+), 52 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index 3811c4cce6d5..66e9e9cbaae2 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -202,7 +202,8 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args types2.StateRo // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, output) + diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -256,7 +257,8 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, output) + diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -386,7 +388,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, output types2.StateNodeSink) (types2.AccountMap, error) { +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { diffAccountAtA := make(types2.AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -420,13 +422,23 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // if this node's path did not show up in diffPathsAtB // that means the node at this path was deleted (or moved) in B // emit an empty "removed" diff to signify as such + // emit emtpy "removed" diff for all storage nodes if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { - if err := output(types2.StateNode{ - Path: node.Path, - NodeValue: []byte{}, + diff := types2.StateNode{ NodeType: types2.Removed, + Path: node.Path, LeafKey: leafKey, - }); err != nil { + NodeValue: []byte{}, + } + + var storageDiffs []types2.StorageNode + err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + if err != nil { + return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) + } + diff.StorageNodes = storageDiffs + + if err := output(diff); err != nil { return nil, err } } @@ -548,7 +560,6 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes } // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator -// if any storage keys are provided it will only return those leaf nodes // including intermediate nodes can be turned on or off func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { for it.Next(true) { @@ -591,6 +602,68 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediate return it.Error() } +// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account +func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Removed Diffs", "root", sr.Hex()) + sTrie, err := sdb.stateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build removed account storage diffs", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator +// including intermediate nodes can be turned on or off +func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + case types2.Extension, types2.Branch: + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go index fb9865f8d684..42fc8488a6f1 100644 --- a/statediff/indexer/database/dump/indexer.go +++ b/statediff/indexer/database/dump/indexer.go @@ -382,10 +382,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -393,25 +394,26 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel) - return err - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node, collect the stateID to reference by FK if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -437,6 +439,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index c842cef12add..d2a87771ee9f 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -392,10 +392,11 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error { // PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -403,23 +404,24 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - sdi.fileWriter.upsertStateCID(stateModel) - return nil - } - stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node sdi.fileWriter.upsertStateCID(stateModel) + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -443,6 +445,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt } sdi.fileWriter.upsertStateAccount(accountModel) } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 790766cfe557..996dbbe1564e 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -440,10 +440,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -451,24 +452,26 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel) - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -494,6 +497,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { From a14745dac57d0e74e53da4f0d3c5840459c72b38 Mon Sep 17 00:00:00 2001 From: nabarun Date: Tue, 15 Mar 2022 19:22:09 +0530 Subject: [PATCH 2/6] Updated unit tests for changes in builder --- statediff/builder_test.go | 152 +++++++++++++++++++++++------- statediff/test_helpers/helpers.go | 12 ++- 2 files changed, 124 insertions(+), 40 deletions(-) diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 5edc1152361f..cf1ea11f6f5b 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -113,7 +113,7 @@ var ( Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), - Root: crypto.Keccak256Hash(slot0StorageLeafRootNode), + Root: crypto.Keccak256Hash(block5StorageBranchRootNode), }) contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), @@ -163,7 +163,7 @@ var ( }) account1AtBlock5, _ = rlp.EncodeToBytes(types.StateAccount{ Nonce: 2, - Balance: big.NewInt(2999566008847709960), + Balance: big.NewInt(2999586469962854280), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) @@ -173,7 +173,7 @@ var ( }) account1AtBlock6, _ = rlp.EncodeToBytes(types.StateAccount{ Nonce: 3, - Balance: big.NewInt(2999537516847709960), + Balance: big.NewInt(2999557977962854280), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) @@ -214,7 +214,7 @@ var ( }) account2AtBlock6, _ = rlp.EncodeToBytes(types.StateAccount{ Nonce: 0, - Balance: big.NewInt(6000063293259748636), + Balance: big.NewInt(6000063258066544204), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) @@ -278,8 +278,8 @@ var ( bankAccountAtBlock4, }) bankAccountAtBlock5, _ = rlp.EncodeToBytes(types.StateAccount{ - Nonce: 7, - Balance: big.NewInt(999805027999990000), + Nonce: 8, + Balance: big.NewInt(999761283999990000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) @@ -460,6 +460,25 @@ var ( []byte{}, []byte{}, }) + block5StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot3StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) ) func init() { @@ -1272,15 +1291,14 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { StorageNodes: []types2.StorageNode{ { Path: []byte{}, - NodeType: types2.Leaf, - NodeValue: slot0StorageLeafRootNode, - LeafKey: slot0StorageKey.Bytes(), + NodeType: types2.Branch, + NodeValue: block5StorageBranchRootNode, }, { - Path: []byte{'\x02'}, - NodeType: types2.Removed, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: []byte{}, + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, }, { Path: []byte{'\x04'}, @@ -1319,11 +1337,29 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { StorageNodes: emptyStorage, }, { - Path: []byte{'\x06'}, - NodeType: types2.Removed, - LeafKey: contractLeafKey, - NodeValue: []byte{}, - StorageNodes: emptyStorage, + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, }, { Path: []byte{'\x0c'}, @@ -1467,16 +1503,10 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. NodeValue: contractAccountAtBlock5LeafNode, StorageNodes: []types2.StorageNode{ { - Path: []byte{}, + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: slot0StorageLeafRootNode, - }, - { - Path: []byte{'\x02'}, - NodeType: types2.Removed, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: []byte{}, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, }, { Path: []byte{'\x04'}, @@ -1513,6 +1543,20 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, }, { Path: []byte{'\x0c'}, @@ -1754,16 +1798,10 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) { NodeValue: contractAccountAtBlock5LeafNode, StorageNodes: []types2.StorageNode{ { - Path: []byte{}, + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: slot0StorageLeafRootNode, - }, - { - Path: []byte{'\x02'}, - NodeType: types2.Removed, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: []byte{}, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, }, { Path: []byte{'\x04'}, @@ -1800,6 +1838,20 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) { NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, }, { Path: []byte{'\x0e'}, @@ -2019,6 +2071,22 @@ func TestBuilderWithMovedAccount(t *testing.T) { NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, }, { Path: []byte{'\x00'}, @@ -2144,6 +2212,18 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, }, { Path: []byte{'\x00'}, diff --git a/statediff/test_helpers/helpers.go b/statediff/test_helpers/helpers.go index 8373f75378cf..39d8081bbdcb 100644 --- a/statediff/test_helpers/helpers.go +++ b/statediff/test_helpers/helpers.go @@ -106,13 +106,17 @@ func TestChainGen(i int, block *core.BlockGen) { block.AddTx(tx3) case 4: // Block 5 has one tx from bankAccount to the contract, that transfers no value - // It sets the remaining storage value to zero + // It sets the one storage value to zero and the other to new value. // Block 5 is mined by Account1Addr block.SetCoinbase(Account1Addr) - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") nonce := block.TxNonce(TestBankAddress) - tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data), signer, TestBankKey) - block.AddTx(tx) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data1), signer, TestBankKey) + nonce++ + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data2), signer, TestBankKey) + block.AddTx(tx1) + block.AddTx(tx2) case 5: // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value // Block 6 is mined by Account2Addr From 84365431d183e9473046ce70f874064b7c3c2ad4 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 16 Mar 2022 11:44:01 +0530 Subject: [PATCH 3/6] Update indexer unit tests --- .../indexer/database/file/indexer_test.go | 98 ++++++++++++------- .../indexer/database/sql/pgx_indexer_test.go | 98 ++++++++++++------- .../indexer/database/sql/sqlx_indexer_test.go | 98 ++++++++++++------- statediff/indexer/mocks/test_data.go | 27 +++++ 4 files changed, 219 insertions(+), 102 deletions(-) diff --git a/statediff/indexer/database/file/indexer_test.go b/statediff/indexer/database/file/indexer_test.go index fb5453fe661a..2b030210d8ae 100644 --- a/statediff/indexer/database/file/indexer_test.go +++ b/statediff/indexer/database/file/indexer_test.go @@ -625,23 +625,34 @@ func TestFileIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(stateNodes), 1) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(stateNodes), 2) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.RemovedLeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } + if idx == 1 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.Contract2LeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x07'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } } - test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) - test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) - test_helpers.ExpectEqual(t, data, []byte{}) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { @@ -694,26 +705,45 @@ func TestFileIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(storageNodes), 1) - test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }) - dc, err = cid.Decode(storageNodes[0].CID) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(storageNodes), 3) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + for idx, storageNode := range storageNodes { + test_helpers.ExpectEqual(t, storageNode, expectedStorageNodes[idx]) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + test_helpers.ExpectEqual(t, data, []byte{}) } - test_helpers.ExpectEqual(t, data, []byte{}) }) } diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index fef559486e8b..94d0e2c4c6c8 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -473,23 +473,34 @@ func TestPGXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(stateNodes), 1) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(stateNodes), 2) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.RemovedLeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } + if idx == 1 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.Contract2LeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x07'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } } - test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) - test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) - test_helpers.ExpectEqual(t, data, []byte{}) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { @@ -541,26 +552,45 @@ func TestPGXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(storageNodes), 1) - test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }) - dc, err = cid.Decode(storageNodes[0].CID) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(storageNodes), 3) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + for idx, storageNode := range storageNodes { + test_helpers.ExpectEqual(t, storageNode, expectedStorageNodes[idx]) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + test_helpers.ExpectEqual(t, data, []byte{}) } - test_helpers.ExpectEqual(t, data, []byte{}) }) } diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index 73309ea7f723..50b855f64f7b 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -466,23 +466,34 @@ func TestSQLXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(stateNodes), 1) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(stateNodes), 2) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.RemovedLeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } + if idx == 1 { + test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) + test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.Contract2LeafKey).Hex()) + test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x07'}) + test_helpers.ExpectEqual(t, data, []byte{}) + } } - test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) - test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) - test_helpers.ExpectEqual(t, data, []byte{}) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { @@ -534,26 +545,45 @@ func TestSQLXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - test_helpers.ExpectEqual(t, len(storageNodes), 1) - test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }) - dc, err = cid.Decode(storageNodes[0].CID) - if err != nil { - t.Fatal(err) + test_helpers.ExpectEqual(t, len(storageNodes), 3) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + for idx, storageNode := range storageNodes { + test_helpers.ExpectEqual(t, storageNode, expectedStorageNodes[idx]) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + test_helpers.ExpectEqual(t, data, []byte{}) } - test_helpers.ExpectEqual(t, data, []byte{}) }) } diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index e5d72e5ba02f..87e80e465bdd 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -57,6 +57,7 @@ var ( Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractAddress2 = crypto.CreateAddress(SenderAddr, MockTransactions[3].Nonce()) MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} mockTopic11 = common.HexToHash("0x04") mockTopic12 = common.HexToHash("0x06") @@ -143,6 +144,12 @@ var ( ContractAccount, }) + Contract2LeafKey = test_helpers.AddressToLeafKey(ContractAddress2) + storage2Location = common.HexToHash("2") + Storage2LeafKey = crypto.Keccak256Hash(storage2Location[:]).Bytes() + storage3Location = common.HexToHash("3") + Storage3LeafKey = crypto.Keccak256Hash(storage3Location[:]).Bytes() + nonce0 = uint64(0) AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") @@ -194,6 +201,26 @@ var ( LeafKey: RemovedLeafKey, NodeValue: []byte{}, }, + { + Path: []byte{'\x07'}, + NodeType: sdtypes.Removed, + LeafKey: Contract2LeafKey, + NodeValue: []byte{}, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Removed, + LeafKey: Storage2LeafKey, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0f'}, + NodeType: sdtypes.Removed, + LeafKey: Storage3LeafKey, + NodeValue: []byte{}, + }, + }, + }, } ) From 9917e6c293f829b6b36cfb599d3cafe66bfa11b7 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 17 Mar 2022 18:04:20 +0530 Subject: [PATCH 4/6] Fix linting errors --- statediff/builder.go | 2 +- statediff/builder_test.go | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index 66e9e9cbaae2..b262f2f8de08 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -422,7 +422,7 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // if this node's path did not show up in diffPathsAtB // that means the node at this path was deleted (or moved) in B // emit an empty "removed" diff to signify as such - // emit emtpy "removed" diff for all storage nodes + // emit empty "removed" diff for all storage nodes if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { diff := types2.StateNode{ NodeType: types2.Removed, diff --git a/statediff/builder_test.go b/statediff/builder_test.go index cf1ea11f6f5b..febf491e515e 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -74,10 +74,6 @@ var ( common.Hex2Bytes("32575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b"), slot3StorageValue, }) - slot0StorageLeafRootNode, _ = rlp.EncodeToBytes([]interface{}{ - common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), - slot0StorageValue, - }) contractAccountAtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ Nonce: 1, From c21aa96962ef5af5b5066ff66b026bb89e8c0282 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 22 Mar 2022 18:04:19 +0530 Subject: [PATCH 5/6] Index Removed diffs without leaf keys if an account or storage node is moved --- statediff/builder.go | 125 ++++++++++++++++++++++++-------------- statediff/builder_test.go | 2 - 2 files changed, 79 insertions(+), 48 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index b262f2f8de08..3de07cd27ee2 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -202,7 +202,7 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args types2.StateRo // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafKeys, params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) @@ -257,7 +257,7 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafKeys, params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) @@ -388,7 +388,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { diffAccountAtA := make(types2.AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -421,23 +421,35 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m } // if this node's path did not show up in diffPathsAtB // that means the node at this path was deleted (or moved) in B - // emit an empty "removed" diff to signify as such - // emit empty "removed" diff for all storage nodes if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { - diff := types2.StateNode{ - NodeType: types2.Removed, - Path: node.Path, - LeafKey: leafKey, - NodeValue: []byte{}, + var diff types2.StateNode + // if this node's leaf key also did not show up in diffAccountsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff state node + // include empty "removed" diff storage nodes for all the storage slots + if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok { + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + LeafKey: leafKey, + NodeValue: []byte{}, + } + + var storageDiffs []types2.StorageNode + err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + if err != nil { + return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) + } + diff.StorageNodes = storageDiffs + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + } } - var storageDiffs []types2.StorageNode - err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) - if err != nil { - return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) - } - diff.StorageNodes = storageDiffs - if err := output(diff); err != nil { return nil, err } @@ -679,22 +691,23 @@ func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common return err } - diffPathsAtB, err := sdb.createdAndUpdatedStorage( + diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), intermediateNodes, output) if err != nil { return err } err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, intermediateNodes, output) + diffSlotsAtB, diffPathsAtB, intermediateNodes, output) if err != nil { return err } return nil } -func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, error) { +func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, map[string]bool, error) { diffPathsAtB := make(map[string]bool) + diffSlotsAtB := make(map[string]bool) it, _ := trie.NewDifferenceIterator(a, b) for it.Next(true) { // skip value nodes @@ -703,7 +716,7 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediat } node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.stateCache.TrieDB()) if err != nil { - return nil, err + return nil, nil, err } switch node.NodeType { case types2.Leaf: @@ -711,13 +724,14 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediat valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] + diffSlotsAtB[common.Bytes2Hex(leafKey)] = true if err := output(types2.StorageNode{ NodeType: node.NodeType, Path: node.Path, NodeValue: node.NodeValue, LeafKey: leafKey, }); err != nil { - return nil, err + return nil, nil, err } case types2.Extension, types2.Branch: if intermediateNodes { @@ -726,18 +740,18 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediat Path: node.Path, NodeValue: node.NodeValue, }); err != nil { - return nil, err + return nil, nil, err } } default: - return nil, fmt.Errorf("unexpected node type %s", node.NodeType) + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) } diffPathsAtB[common.Bytes2Hex(node.Path)] = true } - return diffPathsAtB, it.Error() + return diffSlotsAtB, diffPathsAtB, it.Error() } -func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error { +func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error { it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { // skip value nodes @@ -748,34 +762,53 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB if err != nil { return err } - // if this node path showed up in diffPathsAtB - // that means this node was updated at B and we already have the updated diff for it - // otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event - if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok { - continue - } + switch node.NodeType { case types2.Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - if err := output(types2.StorageNode{ - NodeType: types2.Removed, - Path: node.Path, - NodeValue: []byte{}, - LeafKey: leafKey, - }); err != nil { - return err + + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + // if this node's leaf key also did not show up in diffSlotsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff storage node + if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } } case types2.Extension, types2.Branch: - if intermediateNodes { - if err := output(types2.StorageNode{ - NodeType: types2.Removed, - Path: node.Path, - NodeValue: []byte{}, - }); err != nil { - return err + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted in B + // in that case, emit an empty "removed" diff storage node + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } } } default: diff --git a/statediff/builder_test.go b/statediff/builder_test.go index febf491e515e..70e27ff87119 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -2087,7 +2087,6 @@ func TestBuilderWithMovedAccount(t *testing.T) { { Path: []byte{'\x00'}, NodeType: types2.Removed, - LeafKey: test_helpers.BankLeafKey, NodeValue: []byte{}, }, }, @@ -2224,7 +2223,6 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { { Path: []byte{'\x00'}, NodeType: types2.Removed, - LeafKey: test_helpers.BankLeafKey, NodeValue: []byte{}, }, }, From b351f8f335323f6e123cb7d45b9ee24c17f27086 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 5 Apr 2022 12:38:14 +0530 Subject: [PATCH 6/6] Increase timeout in a failing test --- statediff/service_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statediff/service_test.go b/statediff/service_test.go index 987e1b467f65..2efd3fe51cfa 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -402,7 +402,7 @@ func testGetSyncStatus(t *testing.T) { t.Fatal("Sync Failed") } - time.Sleep(1 * time.Second) + time.Sleep(2 * time.Second) // Make sure if syncStatus is false that WaitForSync has completed! if !syncStatus && len(checkSyncComplete) == 0 {