Merge pull request #5123 from multiversx/add-semi-integration-test

Added semi-integration test

integrationTests/vm/txsFee/asyncESDT_test.go

@@ -6,15 +6,18 @@
 package txsFee
 
 import (
+	"context"
 	"encoding/hex"
 	"math/big"
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/integrationTests/vm"
 	"github.com/multiversx/mx-chain-go/integrationTests/vm/txsFee/utils"
 	"github.com/multiversx/mx-chain-go/process"
+	"github.com/multiversx/mx-chain-go/state"
 	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
 	"github.com/stretchr/testify/require"
 )
@@ -474,3 +477,80 @@ func TestSendNFTToContractWith0FunctionNonPayable(t *testing.T) {
 	_, err = testContext.Accounts.Commit()
 	require.Nil(t, err)
 }
+
+func TestAsyncESDTCallForThirdContractShouldWork(t *testing.T) {
+	testContext, err := vm.CreatePreparedTxProcessorWithVMsMultiShard(0, config.EnableEpochs{})
+	require.Nil(t, err)
+	defer testContext.Close()
+
+	function1 := []byte("add_queued_call")
+	function2 := []byte("forward_queued_calls")
+
+	egldBalance := big.NewInt(100000000)
+	ownerAddr := []byte("owner-78901234567890123456789000")
+	_, _ = vm.CreateAccount(testContext.Accounts, ownerAddr, 0, egldBalance)
+
+	// create an address with ESDT token
+	sndAddr := []byte("sender-8901234567890123456789000")
+
+	esdtBalance := big.NewInt(100000000)
+	esdtTransferValue := big.NewInt(5000)
+	token := []byte("miiutoken")
+	utils.CreateAccountWithESDTBalance(t, testContext.Accounts, sndAddr, egldBalance, token, 0, esdtBalance)
+
+	// deploy contract
+	gasPrice := uint64(10)
+	ownerAccount, _ := testContext.Accounts.LoadAccount(ownerAddr)
+	deployGasLimit := uint64(50000)
+	scAddress := utils.DoDeploySecond(t, testContext, "./testdata/third/third.wasm", ownerAccount, gasPrice, deployGasLimit, nil, big.NewInt(0))
+
+	testContext.TxFeeHandler.CreateBlockStarted(getZeroGasAndFees())
+	utils.CleanAccumulatedIntermediateTransactions(t, testContext)
+
+	// execute first call
+	gasLimit := uint64(500000)
+	tx := utils.CreateESDTTransferTx(0, sndAddr, scAddress, token, esdtTransferValue, gasPrice, gasLimit)
+	tx.Data = []byte(string(tx.Data) + "@" + hex.EncodeToString(function1) + "@01@" + hex.EncodeToString(scAddress) + "@" + hex.EncodeToString(function2))
+
+	retCode, err := testContext.TxProcessor.ProcessTransaction(tx)
+	require.Equal(t, vmcommon.UserError, retCode)
+	require.Nil(t, err)
+
+	utils.CheckESDTBalance(t, testContext, sndAddr, token, esdtBalance)
+	utils.CheckESDTBalance(t, testContext, scAddress, token, big.NewInt(0))
+
+	// execute second call
+	tx = utils.CreateESDTTransferTx(1, sndAddr, scAddress, token, big.NewInt(5000), gasPrice, gasLimit)
+	tx.Data = []byte(string(tx.Data) + "@" + hex.EncodeToString(function2))
+
+	retCode, err = testContext.TxProcessor.ProcessTransaction(tx)
+	require.Equal(t, vmcommon.Ok, retCode)
+	require.Nil(t, err)
+
+	_, err = testContext.Accounts.Commit()
+	require.Nil(t, err)
+
+	utils.CheckESDTBalance(t, testContext, sndAddr, token, big.NewInt(0).Sub(esdtBalance, esdtTransferValue))
+	utils.CheckESDTBalance(t, testContext, scAddress, token, esdtTransferValue)
+
+	// try to recreate the data trie
+	scAccount, err := testContext.Accounts.LoadAccount(scAddress)
+	require.Nil(t, err)
+	userScAccount := scAccount.(state.UserAccountHandler)
+	roothash := userScAccount.GetRootHash()
+	log.Info("recreating data trie", "roothash", roothash)
+
+	leaves := &common.TrieIteratorChannels{
+		LeavesChan: make(chan core.KeyValueHolder, 1),
+		ErrChan:    make(chan error, 1),
+	}
+	err = testContext.Accounts.GetAllLeaves(leaves, context.Background(), roothash)
+	require.Nil(t, err)
+
+	for range leaves.LeavesChan {
+		// do nothing, just iterate
+	}
+
+	err = <-leaves.ErrChan
+	require.Nil(t, err)
+}

state/accountsDB.go

@@ -509,19 +509,22 @@ func saveCodeEntry(codeHash []byte, entry *CodeEntry, trie Updater, marshalizer
 	return nil
 }
 
-// LoadDataTrie retrieves and saves the SC data inside accountHandler object.
+// loadDataTrieConcurrentSafe retrieves and saves the SC data inside accountHandler object.
 // Errors if something went wrong
-func (adb *AccountsDB) loadDataTrie(accountHandler baseAccountHandler, mainTrie common.Trie) error {
-	if len(accountHandler.GetRootHash()) == 0 {
-		return nil
-	}
+func (adb *AccountsDB) loadDataTrieConcurrentSafe(accountHandler baseAccountHandler, mainTrie common.Trie) error {
+	adb.mutOp.Lock()
+	defer adb.mutOp.Unlock()
 
 	dataTrie := adb.dataTries.Get(accountHandler.AddressBytes())
 	if dataTrie != nil {
 		accountHandler.SetDataTrie(dataTrie)
 		return nil
 	}
 
+	if len(accountHandler.GetRootHash()) == 0 {
+		return nil
+	}
+
 	dataTrie, err := mainTrie.Recreate(accountHandler.GetRootHash())
 	if err != nil {
 		return fmt.Errorf("trie was not found for hash, rootHash = %s, err = %w", hex.EncodeToString(accountHandler.GetRootHash()), err)
@@ -702,7 +705,7 @@ func (adb *AccountsDB) LoadAccount(address []byte) (vmcommon.AccountHandler, err
 
 	baseAcc, ok := acnt.(baseAccountHandler)
 	if ok {
-		err = adb.loadDataTrie(baseAcc, mainTrie)
+		err = adb.loadDataTrieConcurrentSafe(baseAcc, mainTrie)
 		if err != nil {
 			return nil, err
 		}
@@ -754,7 +757,7 @@ func (adb *AccountsDB) GetExistingAccount(address []byte) (vmcommon.AccountHandl
 
 	baseAcc, ok := acnt.(baseAccountHandler)
 	if ok {
-		err = adb.loadDataTrie(baseAcc, mainTrie)
+		err = adb.loadDataTrieConcurrentSafe(baseAcc, mainTrie)
 		if err != nil {
 			return nil, err
 		}
@@ -784,7 +787,7 @@ func (adb *AccountsDB) GetAccountFromBytes(address []byte, accountBytes []byte)
 		return acnt, nil
 	}
 
-	err = adb.loadDataTrie(baseAcc, adb.getMainTrie())
+	err = adb.loadDataTrieConcurrentSafe(baseAcc, adb.getMainTrie())
 	if err != nil {
 		return nil, err
 	}
@@ -1099,7 +1102,10 @@ func (adb *AccountsDB) journalize(entry JournalEntry) {
 	}
 
 	adb.entries = append(adb.entries, entry)
-	log.Trace("accountsDB.Journalize", "new length", len(adb.entries))
+	log.Trace("accountsDB.Journalize",
+		"new length", len(adb.entries),
+		"entry type", fmt.Sprintf("%T", entry),
+	)
 
 	if len(adb.entries) == 1 {
 		adb.stackDebug = debug.Stack()
@@ -1146,12 +1152,12 @@ func (adb *AccountsDB) SnapshotState(rootHash []byte) {
 	}
 	stats := newSnapshotStatistics(1, 1)
 
-	accountMetrics := &accountMetrics{
+	accountMetricsInstance := &accountMetrics{
 		snapshotInProgressKey:   common.MetricAccountsSnapshotInProgress,
 		lastSnapshotDurationKey: common.MetricLastAccountsSnapshotDurationSec,
 		snapshotMessage:         userTrieSnapshotMsg,
 	}
-	adb.updateMetricsOnSnapshotStart(accountMetrics)
+	adb.updateMetricsOnSnapshotStart(accountMetricsInstance)
 
 	go func() {
 		stats.NewSnapshotStarted()
@@ -1164,7 +1170,7 @@ func (adb *AccountsDB) SnapshotState(rootHash []byte) {
 
 	go adb.syncMissingNodes(missingNodesChannel, stats, adb.trieSyncer)
 
-	go adb.processSnapshotCompletion(stats, trieStorageManager, missingNodesChannel, iteratorChannels.ErrChan, rootHash, accountMetrics, epoch)
+	go adb.processSnapshotCompletion(stats, trieStorageManager, missingNodesChannel, iteratorChannels.ErrChan, rootHash, accountMetricsInstance, epoch)
 
 	adb.waitForCompletionIfAppropriate(stats)
 }

state/accountsDB_test.go

@@ -698,7 +698,7 @@ func TestAccountsDB_LoadDataNilRootShouldRetNil(t *testing.T) {
 	_, account, adb := generateAddressAccountAccountsDB(tr)
 
 	// since root is nil, result should be nil and data trie should be nil
-	err := adb.LoadDataTrie(account)
+	err := adb.LoadDataTrieConcurrentSafe(account)
 	assert.Nil(t, err)
 	assert.Nil(t, account.DataTrie())
 }
@@ -715,7 +715,7 @@ func TestAccountsDB_LoadDataBadLengthShouldErr(t *testing.T) {
 	account.SetRootHash([]byte("12345"))
 
 	// should return error
-	err := adb.LoadDataTrie(account)
+	err := adb.LoadDataTrieConcurrentSafe(account)
 	assert.NotNil(t, err)
 }
 
@@ -733,7 +733,7 @@ func TestAccountsDB_LoadDataMalfunctionTrieShouldErr(t *testing.T) {
 	adb := generateAccountDBFromTrie(mockTrie)
 
 	// should return error
-	err := adb.LoadDataTrie(account)
+	err := adb.LoadDataTrieConcurrentSafe(account)
 	assert.NotNil(t, err)
 }
 
@@ -751,7 +751,7 @@ func TestAccountsDB_LoadDataNotFoundRootShouldReturnErr(t *testing.T) {
 	account.SetRootHash(rootHash)
 
 	// should return error
-	err := adb.LoadDataTrie(account)
+	err := adb.LoadDataTrieConcurrentSafe(account)
 	assert.NotNil(t, err)
 	fmt.Println(err.Error())
 }
@@ -795,7 +795,7 @@ func TestAccountsDB_LoadDataWithSomeValuesShouldWork(t *testing.T) {
 	account.SetRootHash(rootHash)
 
 	// should not return error
-	err := adb.LoadDataTrie(account)
+	err := adb.LoadDataTrieConcurrentSafe(account)
 	assert.Nil(t, err)
 
 	// verify data
@@ -2812,6 +2812,35 @@ func TestAccountsDb_Concurrent(t *testing.T) {
 	testAccountMethodsConcurrency(t, adb, accountsAddresses, rootHash)
 }
 
+func TestAccountsDB_SaveKeyValAfterAccountIsReverted(t *testing.T) {
+	t.Parallel()
+
+	_, adb := getDefaultTrieAndAccountsDb()
+	addr := generateRandomByteArray(32)
+
+	acc, _ := adb.LoadAccount(addr)
+	_ = adb.SaveAccount(acc)
+
+	acc, _ = adb.LoadAccount(addr)
+	acc.(state.UserAccountHandler).IncreaseNonce(1)
+	_ = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key"), []byte("value"))
+	_ = adb.SaveAccount(acc)
+
+	err := adb.RevertToSnapshot(1)
+	require.Nil(t, err)
+
+	acc, _ = adb.LoadAccount(addr)
+	_ = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key"), []byte("value"))
+	_ = adb.SaveAccount(acc)
+
+	_, err = adb.Commit()
+	require.Nil(t, err)
+
+	acc, err = adb.LoadAccount(addr)
+	require.Nil(t, err)
+	require.NotNil(t, acc)
+}
+
 func testAccountMethodsConcurrency(
 	t *testing.T,
 	adb state.AccountsAdapter,

state/dataTriesHolder.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"github.com/multiversx/mx-chain-go/common"
+	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
 type dataTriesHolder struct {
@@ -20,6 +21,8 @@ func NewDataTriesHolder() *dataTriesHolder {
 
 // Put adds a trie pointer to the tries map
 func (dth *dataTriesHolder) Put(key []byte, tr common.Trie) {
+	log.Trace("put trie in data tries holder", "key", key)
+
 	dth.mutex.Lock()
 	dth.tries[string(key)] = tr
 	dth.mutex.Unlock()
@@ -67,6 +70,13 @@ func (dth *dataTriesHolder) GetAllTries() map[string]common.Trie {
 // Reset clears the tries map
 func (dth *dataTriesHolder) Reset() {
 	dth.mutex.Lock()
+
+	if log.GetLevel() == logger.LogTrace {
+		for key := range dth.tries {
+			log.Trace("reset data tries holder", "key", key)
+		}
+	}
+
 	dth.tries = make(map[string]common.Trie)
 	dth.mutex.Unlock()
 }

state/export_test.go

@@ -22,9 +22,9 @@ func (adb *AccountsDB) LoadCode(accountHandler baseAccountHandler) error {
 	return adb.loadCode(accountHandler)
 }
 
-// LoadDataTrie -
-func (adb *AccountsDB) LoadDataTrie(accountHandler baseAccountHandler) error {
-	return adb.loadDataTrie(accountHandler, adb.getMainTrie())
+// LoadDataTrieConcurrentSafe -
+func (adb *AccountsDB) LoadDataTrieConcurrentSafe(accountHandler baseAccountHandler) error {
+	return adb.loadDataTrieConcurrentSafe(accountHandler, adb.getMainTrie())
 }
 
 // GetAccount -

state/storagePruningManager/storagePruningManager.go

@@ -3,6 +3,7 @@ package storagePruningManager
 import (
 	"bytes"
 	"encoding/hex"
+	"fmt"
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
@@ -81,7 +82,7 @@ func (spm *storagePruningManager) markForEviction(
 		return err
 	}
 
-	logMapWithTrace("MarkForEviction "+string(identifier), "hash", hashes)
+	logMapWithTrace(fmt.Sprintf("MarkForEviction %d", identifier), "hash", hashes)
 	return nil
 }
 

trie/node.go

@@ -5,13 +5,15 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"runtime/debug"
 	"time"
 
 	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/errors"
 	"github.com/multiversx/mx-chain-go/trie/keyBuilder"
+	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
 const (
@@ -119,13 +121,22 @@ func computeAndSetNodeHash(n node) ([]byte, error) {
 func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marshal.Marshalizer, hasher hashing.Hasher) (node, error) {
 	encChild, err := db.Get(n)
 	if err != nil {
-		log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n)
+		treatLogError(log, err, n)
+
 		return nil, fmt.Errorf(common.GetNodeFromDBErrorString+" %w for key %v", err, hex.EncodeToString(n))
 	}
 
 	return decodeNode(encChild, marshalizer, hasher)
 }
 
+func treatLogError(logInstance logger.Logger, err error, key []byte) {
+	if logInstance.GetLevel() != logger.LogTrace {
+		return
+	}
+
+	logInstance.Trace(common.GetNodeFromDBErrorString, "error", err, "key", key, "stack trace", string(debug.Stack()))
+}
+
 func resolveIfCollapsed(n node, pos byte, db common.DBWriteCacher) error {
 	err := n.isEmptyOrNil()
 	if err != nil {