From 4855d9304e80978e8fcf3fafb9d52662bdcd09e4 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 15 Mar 2022 17:10:26 +0530 Subject: [PATCH] Index Removed storage diffs on contract destruction --- statediff/builder.go | 89 ++++++++++++++++++++-- statediff/indexer/database/dump/indexer.go | 33 ++++---- statediff/indexer/database/file/indexer.go | 33 ++++---- statediff/indexer/database/sql/indexer.go | 32 ++++---- 4 files changed, 135 insertions(+), 52 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index 3811c4cce6d5..66e9e9cbaae2 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -202,7 +202,8 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args types2.StateRo // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, output) + diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -256,7 +257,8 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, params.watchedAddressesLeafKeys, output) + diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -386,7 +388,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, output types2.StateNodeSink) (types2.AccountMap, error) { +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { diffAccountAtA := make(types2.AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -420,13 +422,23 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // if this node's path did not show up in diffPathsAtB // that means the node at this path was deleted (or moved) in B // emit an empty "removed" diff to signify as such + // emit emtpy "removed" diff for all storage nodes if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { - if err := output(types2.StateNode{ - Path: node.Path, - NodeValue: []byte{}, + diff := types2.StateNode{ NodeType: types2.Removed, + Path: node.Path, LeafKey: leafKey, - }); err != nil { + NodeValue: []byte{}, + } + + var storageDiffs []types2.StorageNode + err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + if err != nil { + return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) + } + diff.StorageNodes = storageDiffs + + if err := output(diff); err != nil { return nil, err } } @@ -548,7 +560,6 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes } // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator -// if any storage keys are provided it will only return those leaf nodes // including intermediate nodes can be turned on or off func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { for it.Next(true) { @@ -591,6 +602,68 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediate return it.Error() } +// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account +func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Removed Diffs", "root", sr.Hex()) + sTrie, err := sdb.stateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build removed account storage diffs", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator +// including intermediate nodes can be turned on or off +func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + case types2.Extension, types2.Branch: + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go index fb9865f8d684..42fc8488a6f1 100644 --- a/statediff/indexer/database/dump/indexer.go +++ b/statediff/indexer/database/dump/indexer.go @@ -382,10 +382,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -393,25 +394,26 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel) - return err - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node, collect the stateID to reference by FK if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -437,6 +439,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index c842cef12add..d2a87771ee9f 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -392,10 +392,11 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error { // PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -403,23 +404,24 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - sdi.fileWriter.upsertStateCID(stateModel) - return nil - } - stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node sdi.fileWriter.upsertStateCID(stateModel) + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -443,6 +445,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt } sdi.fileWriter.upsertStateAccount(accountModel) } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 790766cfe557..996dbbe1564e 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -440,10 +440,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -451,24 +452,26 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel) - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -494,6 +497,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed {