From c94918b7411cbcf19e3322be959a6ba193628028 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Tue, 4 Oct 2022 13:00:57 -0600
Subject: [PATCH 01/10] Support recreating referenced state and simplify
 reference management

---
 arbitrum/recordingdb.go | 63 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index d3147807d0..e820035c65 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -2,6 +2,7 @@ package arbitrum
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -12,6 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -193,3 +195,64 @@ func PreimagesFromRecording(chainContextIf core.ChainContext, recordingDb *Recor
 	}
 	return entries, nil
 }
+
+func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc *core.BlockChain) (*state.StateDB, error) {
+	stateDatbase := bc.StateCache()
+	trieDB := stateDatbase.TrieDB()
+	stateDb, err := state.New(header.Root, stateDatbase, nil)
+	if err == nil {
+		trieDB.Reference(header.Root, common.Hash{})
+		return stateDb, nil
+	}
+	currentHeader := header
+	var lastRoot common.Hash
+	for ctx.Err() == nil {
+		nextHeader := bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
+		if nextHeader == nil {
+			return nil, fmt.Errorf("chain doesn't contain parent of block %d hash %v", currentHeader.Number, currentHeader.Hash())
+		}
+		currentHeader = nextHeader
+		if currentHeader.Number.Uint64() < bc.Config().ArbitrumChainParams.GenesisBlockNum {
+			return nil, fmt.Errorf("moved beyond genesis looking for state")
+		}
+		stateDb, err = state.New(currentHeader.Root, stateDatbase, nil)
+		if err == nil {
+			trieDB.Reference(header.Root, common.Hash{})
+			lastRoot = currentHeader.Root
+			defer func() { trieDB.Dereference(lastRoot) }()
+			break
+		}
+	}
+	returnedBlockNumber := header.Number.Uint64()
+	blockToRecreate := currentHeader.Number.Uint64() + 1
+	for ctx.Err() == nil {
+		block := bc.GetBlockByNumber(blockToRecreate)
+		if block == nil {
+			return nil, fmt.Errorf("block not found while recreating: %d", blockToRecreate)
+		}
+		_, _, _, err := bc.Processor().Process(block, stateDb, vm.Config{})
+		if err != nil {
+			return nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err)
+		}
+		trieDB.Reference(block.Root(), common.Hash{})
+		lastRoot = block.Root()
+		if blockToRecreate >= returnedBlockNumber {
+			if block.Hash() != header.Hash() {
+				return nil, fmt.Errorf("blockHash doesn't match when recreating number: %d expected: %v got: %v", blockToRecreate, header.Hash(), block.Hash())
+			}
+			// double reference because te defer is going to remove one
+			trieDB.Reference(block.Root(), common.Hash{})
+			return stateDb, nil
+		}
+		blockToRecreate++
+	}
+	return nil, ctx.Err()
+}
+
+func ReferenceState(header *types.Header, bc *core.BlockChain) {
+	bc.StateCache().TrieDB().Reference(header.Root, common.Hash{})
+}
+
+func DereferenceState(header *types.Header, bc *core.BlockChain) {
+	bc.StateCache().TrieDB().Dereference(header.Root)
+}

From 2ee2168d5820f2551c9bd192d9ecfa7818919e08 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Tue, 18 Oct 2022 23:21:49 +0000
Subject: [PATCH 02/10] recordings use a separate state database

---
 arbitrum/recordingdb.go | 45 +++++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index e820035c65..3f45c3ee3b 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -151,9 +151,8 @@ func (r *RecordingChainContext) GetMinBlockNumberAccessed() uint64 {
 	return r.minBlockNumberAccessed
 }
 
-func PrepareRecording(blockchain *core.BlockChain, lastBlockHeader *types.Header) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
-	rawTrie := blockchain.StateCache().TrieDB()
-	recordingKeyValue := NewRecordingKV(rawTrie)
+func PrepareRecording(trieDB *trie.Database, chainContext core.ChainContext, lastBlockHeader *types.Header) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
+	recordingKeyValue := NewRecordingKV(trieDB)
 
 	recordingStateDatabase := state.NewDatabase(rawdb.NewDatabase(recordingKeyValue))
 	var prevRoot common.Hash
@@ -169,7 +168,7 @@ func PrepareRecording(blockchain *core.BlockChain, lastBlockHeader *types.Header
 		if !lastBlockHeader.Number.IsUint64() {
 			return nil, nil, nil, errors.New("block number not uint64")
 		}
-		recordingChainContext = NewRecordingChainContext(blockchain, lastBlockHeader.Number.Uint64())
+		recordingChainContext = NewRecordingChainContext(chainContext, lastBlockHeader.Number.Uint64())
 	}
 	return recordingStateDb, recordingChainContext, recordingKeyValue, nil
 }
@@ -196,26 +195,26 @@ func PreimagesFromRecording(chainContextIf core.ChainContext, recordingDb *Recor
 	return entries, nil
 }
 
-func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc *core.BlockChain) (*state.StateDB, error) {
-	stateDatbase := bc.StateCache()
-	trieDB := stateDatbase.TrieDB()
-	stateDb, err := state.New(header.Root, stateDatbase, nil)
+func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc *core.BlockChain, stateDatabase state.Database) (*state.StateDB, error) {
+	trieDB := stateDatabase.TrieDB()
+	stateDb, err := state.New(header.Root, stateDatabase, nil)
 	if err == nil {
 		trieDB.Reference(header.Root, common.Hash{})
 		return stateDb, nil
 	}
+	returnedBlockNumber := header.Number.Uint64()
+	genesis := bc.Config().ArbitrumChainParams.GenesisBlockNum
 	currentHeader := header
 	var lastRoot common.Hash
 	for ctx.Err() == nil {
-		nextHeader := bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
-		if nextHeader == nil {
-			return nil, fmt.Errorf("chain doesn't contain parent of block %d hash %v", currentHeader.Number, currentHeader.Hash())
+		if currentHeader.Number.Uint64() <= genesis {
+			return nil, fmt.Errorf("moved beyond genesis looking for state looking for %d, genesis %d, err %w", returnedBlockNumber, genesis, err)
 		}
-		currentHeader = nextHeader
-		if currentHeader.Number.Uint64() < bc.Config().ArbitrumChainParams.GenesisBlockNum {
-			return nil, fmt.Errorf("moved beyond genesis looking for state")
+		currentHeader = bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
+		if currentHeader == nil {
+			return nil, fmt.Errorf("chain doesn't contain parent of block %d hash %v", currentHeader.Number, currentHeader.Hash())
 		}
-		stateDb, err = state.New(currentHeader.Root, stateDatbase, nil)
+		stateDb, err = state.New(currentHeader.Root, stateDatabase, nil)
 		if err == nil {
 			trieDB.Reference(header.Root, common.Hash{})
 			lastRoot = currentHeader.Root
@@ -223,7 +222,6 @@ func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc
 			break
 		}
 	}
-	returnedBlockNumber := header.Number.Uint64()
 	blockToRecreate := currentHeader.Number.Uint64() + 1
 	for ctx.Err() == nil {
 		block := bc.GetBlockByNumber(blockToRecreate)
@@ -234,6 +232,13 @@ func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc
 		if err != nil {
 			return nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err)
 		}
+		root, err := stateDb.Commit(true)
+		if err != nil {
+			return nil, fmt.Errorf("failed commiting state for block %d : %w", blockToRecreate, err)
+		}
+		if root != block.Root() {
+			return nil, fmt.Errorf("bad state recreating block %d : exp: %v got %v", blockToRecreate, block.Root(), root)
+		}
 		trieDB.Reference(block.Root(), common.Hash{})
 		lastRoot = block.Root()
 		if blockToRecreate >= returnedBlockNumber {
@@ -249,10 +254,10 @@ func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc
 	return nil, ctx.Err()
 }
 
-func ReferenceState(header *types.Header, bc *core.BlockChain) {
-	bc.StateCache().TrieDB().Reference(header.Root, common.Hash{})
+func ReferenceState(header *types.Header, stateDatabase state.Database) {
+	stateDatabase.TrieDB().Reference(header.Root, common.Hash{})
 }
 
-func DereferenceState(header *types.Header, bc *core.BlockChain) {
-	bc.StateCache().TrieDB().Dereference(header.Root)
+func DereferenceState(header *types.Header, stateDatabase state.Database) {
+	stateDatabase.TrieDB().Dereference(header.Root)
 }

From d33910fae37afbc9b3ad4a53ccba29093477717c Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Thu, 20 Oct 2022 17:34:58 -0600
Subject: [PATCH 03/10] use recordingDatabase

It simplifies recording data and takes care of references
---
 arbitrum/recordingdb.go | 130 ++++++++++++++++++++++++++++------------
 1 file changed, 92 insertions(+), 38 deletions(-)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index 3f45c3ee3b..ca9674400c 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -27,7 +28,7 @@ type RecordingKV struct {
 	enableBypass  bool
 }
 
-func NewRecordingKV(inner *trie.Database) *RecordingKV {
+func newRecordingKV(inner *trie.Database) *RecordingKV {
 	return &RecordingKV{inner, make(map[common.Hash][]byte), false}
 }
 
@@ -128,7 +129,7 @@ type RecordingChainContext struct {
 	initialBlockNumber     uint64
 }
 
-func NewRecordingChainContext(inner core.ChainContext, blocknumber uint64) *RecordingChainContext {
+func newRecordingChainContext(inner core.ChainContext, blocknumber uint64) *RecordingChainContext {
 	return &RecordingChainContext{
 		bc:                     inner,
 		minBlockNumberAccessed: blocknumber,
@@ -151,8 +152,74 @@ func (r *RecordingChainContext) GetMinBlockNumberAccessed() uint64 {
 	return r.minBlockNumberAccessed
 }
 
-func PrepareRecording(trieDB *trie.Database, chainContext core.ChainContext, lastBlockHeader *types.Header) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
-	recordingKeyValue := NewRecordingKV(trieDB)
+type RecordingDatabase struct {
+	db         state.Database
+	bc         *core.BlockChain
+	mutex      sync.Mutex // protects StateFor and Dereference
+	references int64
+}
+
+func NewRecordingDatabase(ethdb ethdb.Database, blockchain *core.BlockChain) *RecordingDatabase {
+	return &RecordingDatabase{
+		db: state.NewDatabaseWithConfig(ethdb, &trie.Config{Cache: 16}), //TODO cache needed? configurable?
+		bc: blockchain,
+	}
+}
+
+// Normal geth state.New + Reference is not atomic vs Dereference. This one is.
+// This function does not recreate a state
+func (r *RecordingDatabase) StateFor(header *types.Header) (*state.StateDB, error) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	sdb, err := state.NewDeterministic(header.Root, r.db)
+	if err == nil {
+		r.referenceRootLockHeld(header.Root)
+	}
+	return sdb, err
+}
+
+func (r *RecordingDatabase) Dereference(header *types.Header) {
+	if header != nil {
+		r.dereferenceRoot(header.Root)
+	}
+}
+
+// lock must be held when calling that
+func (r *RecordingDatabase) referenceRootLockHeld(root common.Hash) {
+	r.references++
+	r.db.TrieDB().Reference(root, common.Hash{})
+}
+
+func (r *RecordingDatabase) dereferenceRoot(root common.Hash) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.references--
+	r.db.TrieDB().Dereference(root)
+}
+
+func (r *RecordingDatabase) addStateVerify(statedb *state.StateDB, expected common.Hash) error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	result, err := statedb.Commit(true)
+	if err != nil {
+		return err
+	}
+	if result != expected {
+		return fmt.Errorf("bad root hash expected: %v got: %v", expected, result)
+	}
+	r.referenceRootLockHeld(result)
+	return nil
+}
+
+func (r *RecordingDatabase) PrepareRecording(ctx context.Context, lastBlockHeader *types.Header) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
+	_, err := r.GetOrRecreateState(ctx, lastBlockHeader)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	finalDereference := lastBlockHeader // dereference in case of error
+	defer func() { r.Dereference(finalDereference) }()
+	recordingKeyValue := newRecordingKV(r.db.TrieDB())
 
 	recordingStateDatabase := state.NewDatabase(rawdb.NewDatabase(recordingKeyValue))
 	var prevRoot common.Hash
@@ -168,96 +235,83 @@ func PrepareRecording(trieDB *trie.Database, chainContext core.ChainContext, las
 		if !lastBlockHeader.Number.IsUint64() {
 			return nil, nil, nil, errors.New("block number not uint64")
 		}
-		recordingChainContext = NewRecordingChainContext(chainContext, lastBlockHeader.Number.Uint64())
+		recordingChainContext = newRecordingChainContext(r.bc, lastBlockHeader.Number.Uint64())
 	}
+	finalDereference = nil
 	return recordingStateDb, recordingChainContext, recordingKeyValue, nil
 }
 
-func PreimagesFromRecording(chainContextIf core.ChainContext, recordingDb *RecordingKV) (map[common.Hash][]byte, error) {
+func (r *RecordingDatabase) PreimagesFromRecording(chainContextIf core.ChainContext, recordingDb *RecordingKV) (map[common.Hash][]byte, error) {
 	entries := recordingDb.GetRecordedEntries()
 	recordingChainContext, ok := chainContextIf.(*RecordingChainContext)
 	if (recordingChainContext == nil) || (!ok) {
 		return nil, errors.New("recordingChainContext invalid")
 	}
-	blockchain, ok := recordingChainContext.bc.(*core.BlockChain)
-	if (blockchain == nil) || (!ok) {
-		return nil, errors.New("blockchain invalid")
-	}
+
 	for i := recordingChainContext.GetMinBlockNumberAccessed(); i <= recordingChainContext.initialBlockNumber; i++ {
-		header := blockchain.GetHeaderByNumber(i)
+		header := r.bc.GetHeaderByNumber(i)
 		hash := header.Hash()
 		bytes, err := rlp.EncodeToBytes(header)
 		if err != nil {
-			panic(fmt.Sprintf("Error RLP encoding header: %v\n", err))
+			return nil, fmt.Errorf("Error RLP encoding header: %v\n", err)
 		}
 		entries[hash] = bytes
 	}
 	return entries, nil
 }
 
-func GetOrRecreateReferencedState(ctx context.Context, header *types.Header, bc *core.BlockChain, stateDatabase state.Database) (*state.StateDB, error) {
-	trieDB := stateDatabase.TrieDB()
-	stateDb, err := state.New(header.Root, stateDatabase, nil)
+func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *types.Header) (*state.StateDB, error) {
+	stateDb, err := r.StateFor(header)
 	if err == nil {
-		trieDB.Reference(header.Root, common.Hash{})
 		return stateDb, nil
 	}
 	returnedBlockNumber := header.Number.Uint64()
-	genesis := bc.Config().ArbitrumChainParams.GenesisBlockNum
+	genesis := r.bc.Config().ArbitrumChainParams.GenesisBlockNum
 	currentHeader := header
 	var lastRoot common.Hash
 	for ctx.Err() == nil {
 		if currentHeader.Number.Uint64() <= genesis {
 			return nil, fmt.Errorf("moved beyond genesis looking for state looking for %d, genesis %d, err %w", returnedBlockNumber, genesis, err)
 		}
-		currentHeader = bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
+		currentHeader = r.bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
 		if currentHeader == nil {
 			return nil, fmt.Errorf("chain doesn't contain parent of block %d hash %v", currentHeader.Number, currentHeader.Hash())
 		}
-		stateDb, err = state.New(currentHeader.Root, stateDatabase, nil)
+		stateDb, err = r.StateFor(currentHeader)
 		if err == nil {
-			trieDB.Reference(header.Root, common.Hash{})
 			lastRoot = currentHeader.Root
-			defer func() { trieDB.Dereference(lastRoot) }()
+			defer func() {
+				if (lastRoot != common.Hash{}) {
+					r.dereferenceRoot(lastRoot)
+				}
+			}()
 			break
 		}
 	}
 	blockToRecreate := currentHeader.Number.Uint64() + 1
 	for ctx.Err() == nil {
-		block := bc.GetBlockByNumber(blockToRecreate)
+		block := r.bc.GetBlockByNumber(blockToRecreate)
 		if block == nil {
 			return nil, fmt.Errorf("block not found while recreating: %d", blockToRecreate)
 		}
-		_, _, _, err := bc.Processor().Process(block, stateDb, vm.Config{})
+		_, _, _, err := r.bc.Processor().Process(block, stateDb, vm.Config{})
 		if err != nil {
 			return nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err)
 		}
-		root, err := stateDb.Commit(true)
+		err = r.addStateVerify(stateDb, block.Root())
 		if err != nil {
 			return nil, fmt.Errorf("failed commiting state for block %d : %w", blockToRecreate, err)
 		}
-		if root != block.Root() {
-			return nil, fmt.Errorf("bad state recreating block %d : exp: %v got %v", blockToRecreate, block.Root(), root)
-		}
-		trieDB.Reference(block.Root(), common.Hash{})
 		lastRoot = block.Root()
 		if blockToRecreate >= returnedBlockNumber {
 			if block.Hash() != header.Hash() {
 				return nil, fmt.Errorf("blockHash doesn't match when recreating number: %d expected: %v got: %v", blockToRecreate, header.Hash(), block.Hash())
 			}
-			// double reference because te defer is going to remove one
-			trieDB.Reference(block.Root(), common.Hash{})
+			// don't dereference this one
+			lastRoot = common.Hash{}
 			return stateDb, nil
 		}
 		blockToRecreate++
 	}
 	return nil, ctx.Err()
 }
-
-func ReferenceState(header *types.Header, stateDatabase state.Database) {
-	stateDatabase.TrieDB().Reference(header.Root, common.Hash{})
-}
-
-func DereferenceState(header *types.Header, stateDatabase state.Database) {
-	stateDatabase.TrieDB().Dereference(header.Root)
-}

From 1be94888132ed2ed3d66a616d253562e586731d2 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Fri, 21 Oct 2022 12:50:04 -0600
Subject: [PATCH 04/10] add LogFunc to state recalculation functions

---
 arbitrum/recordingdb.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index ca9674400c..8c0e6b3d1d 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -212,8 +212,10 @@ func (r *RecordingDatabase) addStateVerify(statedb *state.StateDB, expected comm
 	return nil
 }
 
-func (r *RecordingDatabase) PrepareRecording(ctx context.Context, lastBlockHeader *types.Header) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
-	_, err := r.GetOrRecreateState(ctx, lastBlockHeader)
+type StateBuildingLogFunction func(targetHeader, header *types.Header, hasState bool)
+
+func (r *RecordingDatabase) PrepareRecording(ctx context.Context, lastBlockHeader *types.Header, logFunc StateBuildingLogFunction) (*state.StateDB, core.ChainContext, *RecordingKV, error) {
+	_, err := r.GetOrRecreateState(ctx, lastBlockHeader, logFunc)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -260,7 +262,7 @@ func (r *RecordingDatabase) PreimagesFromRecording(chainContextIf core.ChainCont
 	return entries, nil
 }
 
-func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *types.Header) (*state.StateDB, error) {
+func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *types.Header, logFunc StateBuildingLogFunction) (*state.StateDB, error) {
 	stateDb, err := r.StateFor(header)
 	if err == nil {
 		return stateDb, nil
@@ -270,6 +272,9 @@ func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *type
 	currentHeader := header
 	var lastRoot common.Hash
 	for ctx.Err() == nil {
+		if logFunc != nil {
+			logFunc(header, currentHeader, false)
+		}
 		if currentHeader.Number.Uint64() <= genesis {
 			return nil, fmt.Errorf("moved beyond genesis looking for state looking for %d, genesis %d, err %w", returnedBlockNumber, genesis, err)
 		}
@@ -294,6 +299,9 @@ func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *type
 		if block == nil {
 			return nil, fmt.Errorf("block not found while recreating: %d", blockToRecreate)
 		}
+		if logFunc != nil {
+			logFunc(header, block.Header(), true)
+		}
 		_, _, _, err := r.bc.Processor().Process(block, stateDb, vm.Config{})
 		if err != nil {
 			return nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err)

From 0f41a80d8aecf5b0b4375b51ead97180e7ea70d1 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Fri, 21 Oct 2022 13:49:58 -0600
Subject: [PATCH 05/10] fix deref bug and add visibility

---
 arbitrum/recordingdb.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index 8c0e6b3d1d..9788333a7b 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -310,6 +310,7 @@ func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *type
 		if err != nil {
 			return nil, fmt.Errorf("failed commiting state for block %d : %w", blockToRecreate, err)
 		}
+		r.dereferenceRoot(lastRoot)
 		lastRoot = block.Root()
 		if blockToRecreate >= returnedBlockNumber {
 			if block.Hash() != header.Hash() {
@@ -323,3 +324,7 @@ func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *type
 	}
 	return nil, ctx.Err()
 }
+
+func (r *RecordingDatabase) ReferenceCount() int64 {
+	return r.references
+}

From 9ad5828dc6e9ef041387c02e0c0bc00562e81b7f Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Tue, 25 Oct 2022 14:04:23 -0600
Subject: [PATCH 06/10] recordingDb: support writing to HD

---
 arbitrum/recordingdb.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index 9788333a7b..cf1c99dc16 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -185,6 +185,13 @@ func (r *RecordingDatabase) Dereference(header *types.Header) {
 	}
 }
 
+func (r *RecordingDatabase) WriteStateToDatabase(header *types.Header) error {
+	if header != nil {
+		return r.db.TrieDB().Commit(header.Root, true, nil)
+	}
+	return nil
+}
+
 // lock must be held when calling that
 func (r *RecordingDatabase) referenceRootLockHeld(root common.Hash) {
 	r.references++

From d5c689f22f3a612dfb74f617ce04530b8e4f0956 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Sun, 30 Oct 2022 14:40:10 +0000
Subject: [PATCH 07/10] stateDatabase: add finalizer to free codecache

---
 core/state/database.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/core/state/database.go b/core/state/database.go
index ce5d8d7317..57ecb4a473 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -19,6 +19,7 @@ package state
 import (
 	"errors"
 	"fmt"
+	"runtime"
 
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
@@ -118,11 +119,13 @@ func NewDatabase(db ethdb.Database) Database {
 // large memory cache.
 func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
-	return &cachingDB{
+	cdb := &cachingDB{
 		db:            trie.NewDatabaseWithConfig(db, config),
 		codeSizeCache: csc,
 		codeCache:     fastcache.New(codeCacheSize),
 	}
+	runtime.SetFinalizer(cdb, (*cachingDB).finalizer)
+	return cdb
 }
 
 type cachingDB struct {
@@ -140,6 +143,11 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 	return tr, nil
 }
 
+// OpenTrie opens the main account trie at a specific root hash.
+func (db *cachingDB) finalizer() {
+	db.codeCache.Reset()
+}
+
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
 	tr, err := trie.NewSecure(addrHash, root, db.db)

From 5c479f09264462be1edae8e2e7650f473f545928 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Mon, 31 Oct 2022 08:39:49 -0600
Subject: [PATCH 08/10] Fix comment

---
 core/state/database.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/state/database.go b/core/state/database.go
index 57ecb4a473..eaaf5606c3 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -143,7 +143,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 	return tr, nil
 }
 
-// OpenTrie opens the main account trie at a specific root hash.
+// fastcache chunks are not mannaged by GC.
 func (db *cachingDB) finalizer() {
 	db.codeCache.Reset()
 }

From b4df61d0530fe335759834ad326d7437792bfdd6 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Sun, 30 Oct 2022 11:58:55 -0600
Subject: [PATCH 09/10] trie: add finalizer to free memory from cleans
 fastcache

---
 trie/database.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/trie/database.go b/trie/database.go
index 9ac6dcec5c..650dbb81c2 100644
--- a/trie/database.go
+++ b/trie/database.go
@@ -297,6 +297,7 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database
 	if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future
 		db.preimages = make(map[common.Hash][]byte)
 	}
+	runtime.SetFinalizer(db, (*Database).finalizer)
 	return db
 }
 
@@ -305,6 +306,13 @@ func (db *Database) DiskDB() ethdb.KeyValueStore {
 	return db.diskdb
 }
 
+// must call Reset() to reclaim memory used by fastcache
+func (db *Database) finalizer() {
+	if db.cleans != nil {
+		db.cleans.Reset()
+	}
+}
+
 // insert inserts a collapsed trie node into the memory database.
 // The blob size must be specified to allow proper size tracking.
 // All nodes inserted by this function will be reference tracked

From ccadba06677a395b49918822a3e93610a64aac3d Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <tsahi@offchainlabs.com>
Date: Wed, 2 Nov 2022 17:11:07 -0600
Subject: [PATCH 10/10] rcordingdb: address comments

---
 arbitrum/recordingdb.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/arbitrum/recordingdb.go b/arbitrum/recordingdb.go
index cf1c99dc16..a6f9818896 100644
--- a/arbitrum/recordingdb.go
+++ b/arbitrum/recordingdb.go
@@ -292,20 +292,25 @@ func (r *RecordingDatabase) GetOrRecreateState(ctx context.Context, header *type
 		stateDb, err = r.StateFor(currentHeader)
 		if err == nil {
 			lastRoot = currentHeader.Root
-			defer func() {
-				if (lastRoot != common.Hash{}) {
-					r.dereferenceRoot(lastRoot)
-				}
-			}()
 			break
 		}
 	}
+	defer func() {
+		if (lastRoot != common.Hash{}) {
+			r.dereferenceRoot(lastRoot)
+		}
+	}()
 	blockToRecreate := currentHeader.Number.Uint64() + 1
+	prevHash := currentHeader.Hash()
 	for ctx.Err() == nil {
 		block := r.bc.GetBlockByNumber(blockToRecreate)
 		if block == nil {
 			return nil, fmt.Errorf("block not found while recreating: %d", blockToRecreate)
 		}
+		if block.ParentHash() != prevHash {
+			return nil, fmt.Errorf("reorg detected: number %d expectedPrev: %v foundPrev: %v", blockToRecreate, prevHash, block.ParentHash())
+		}
+		prevHash = block.Hash()
 		if logFunc != nil {
 			logFunc(header, block.Header(), true)
 		}