feat: implement reconciliation onchain

.github/workflows/integration.yml

@@ -27,7 +27,7 @@ jobs:
       # used to debug workflow
 #      - name: Setup tmate session
 #        uses: mxschmitt/action-tmate@v3
-      - run: make bep159_integration_test integration_test
+      - run: make bep159_integration_test recon_integration_test integration_test
   coverage-test:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -26,3 +26,6 @@ app/apptest/data
 app_test/data
 plugins/param/data
 plugins/tokens/data
+
+# e2e temp files
+e2e/priv_validator_key.json

Makefile

@@ -205,9 +205,14 @@ integration_test: build
 bep159_integration_test: build
 	@echo "-->BEP159 Integration Test"
 	@bash ./scripts/bep159_integration_test.sh
+
+recon_integration_test: build
+	@echo "-->Recon Integration Test"
+	@bash ./scripts/recon_integration_test.sh
+
 ########################################
 ### Pre Commit
-pre_commit: build test_unit bep159_integration_test integration_test format lint multi-nodes-test
+pre_commit: build test_unit recon_integration_test bep159_integration_test integration_test format lint multi-nodes-test
 
 ########################################
 ### Local validator nodes using docker and docker-compose

app/app.go

@@ -258,6 +258,7 @@ func NewBinanceChain(logger log.Logger, db dbm.DB, traceStore io.Writer, baseApp
 		common.BridgeStoreKey,
 		common.OracleStoreKey,
 		common.IbcStoreKey,
+		common.ReconStoreKey,
 	)
 	app.SetAnteHandler(tx.NewAnteHandler(app.AccountKeeper))
 	app.SetPreChecker(tx.NewTxPreChecker())
@@ -269,6 +270,18 @@ func NewBinanceChain(logger log.Logger, db dbm.DB, traceStore io.Writer, baseApp
 		cmn.Exit(err.Error())
 	}
 
+	// enable diff for reconciliation
+	accountIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
+	if !ok {
+		cmn.Exit("cannot convert account store to ival store")
+	}
+	accountIavl.EnableDiff()
+	tokenIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
+	if !ok {
+		cmn.Exit("cannot convert token store to ival store")
+	}
+	tokenIavl.EnableDiff()
+
 	// init app cache
 	accountStore := app.BaseApp.GetCommitMultiStore().GetKVStore(common.AccountStoreKey)
 	app.SetAccountStoreCache(cdc, accountStore, app.baseConfig.AccountCacheSize)
@@ -345,13 +358,16 @@ func SetUpgradeConfig(upgradeConfig *config.UpgradeConfig) {
 	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP173, upgradeConfig.BEP173Height)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.FixDoubleSignChainId, upgradeConfig.FixDoubleSignChainIdHeight)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP126, upgradeConfig.BEP126Height)
+	upgrade.Mgr.AddUpgradeHeight(upgrade.EnableReconciliation, upgradeConfig.EnableReconciliationHeight)
+	upgrade.Mgr.AddUpgradeHeight(upgrade.DisableMessagesPhase1, upgradeConfig.DisableMessagesPhase1Height)
 
 	// register store keys of upgrade
 	upgrade.Mgr.RegisterStoreKeys(upgrade.BEP9, common.TimeLockStoreKey.Name())
 	upgrade.Mgr.RegisterStoreKeys(upgrade.BEP3, common.AtomicSwapStoreKey.Name())
 	upgrade.Mgr.RegisterStoreKeys(upgrade.LaunchBscUpgrade, common.IbcStoreKey.Name(), common.SideChainStoreKey.Name(),
 		common.SlashingStoreKey.Name(), common.BridgeStoreKey.Name(), common.OracleStoreKey.Name())
 	upgrade.Mgr.RegisterStoreKeys(upgrade.BEP128, common.StakeRewardStoreKey.Name())
+	upgrade.Mgr.RegisterStoreKeys(upgrade.EnableReconciliation, common.ReconStoreKey.Name())
 
 	// register msg types of upgrade
 	upgrade.Mgr.RegisterMsgTypes(upgrade.BEP9,
@@ -953,6 +969,11 @@ func (app *BinanceChain) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) a
 	pub.Pool.Clean()
 	// match may end with transaction failure, which is better to save into
 	// the EndBlock response. However, current cosmos doesn't support this.
+
+	if sdk.IsUpgrade(upgrade.EnableReconciliation) {
+		app.reconBalance(ctx)
+	}
+
 	return abci.ResponseEndBlock{
 		ValidatorUpdates: validatorUpdates,
 		Events:           ctx.EventManager().ABCIEvents(),

app/config/config.go

@@ -101,6 +101,10 @@ BEP173Height = {{ .UpgradeConfig.BEP173Height }}
 FixDoubleSignChainIdHeight = {{ .UpgradeConfig.FixDoubleSignChainIdHeight }}
 # Block height of BEP126 upgrade
 BEP126Height = {{ .UpgradeConfig.BEP126Height }}
+# Block height of reconciliation upgrade
+EnableReconciliationHeight = {{ .UpgradeConfig.EnableReconciliationHeight }}
+# Block height of disable messages phase1 upgrade
+DisableMessagesPhase1Height = {{ .UpgradeConfig.DisableMessagesPhase1Height }}
 
 [query]
 # ABCI query interface black list, suggested value: ["custom/gov/proposals", "custom/timelock/timelocks", "custom/atomicSwap/swapcreator", "custom/atomicSwap/swaprecipient"]
@@ -549,6 +553,8 @@ type UpgradeConfig struct {
 	BEP173Height                                    int64 `mapstructure:"BEP173Height"`
 	FixDoubleSignChainIdHeight                      int64 `mapstructure:"FixDoubleSignChainIdHeight"`
 	BEP126Height                                    int64 `mapstructure:"BEP126Height"`
+	EnableReconciliationHeight                      int64 `mapstructure:"EnableReconciliationHeight"`
+	DisableMessagesPhase1Height                     int64 `mapstructure:"DisableMessagesPhase1Height"`
 }
 
 func defaultUpgradeConfig() *UpgradeConfig {
@@ -569,7 +575,7 @@ func defaultUpgradeConfig() *UpgradeConfig {
 		BEP70Height:                       1,
 		LaunchBscUpgradeHeight:            1,
 		LimitConsAddrUpdateIntervalHeight: math.MaxInt64,
-		BEP126Height:               	   math.MaxInt64,
+		BEP126Height:                      math.MaxInt64,
 		BEP128Height:                      math.MaxInt64,
 		BEP151Height:                      math.MaxInt64,
 		BEP153Height:                      math.MaxInt64,
@@ -583,6 +589,8 @@ func defaultUpgradeConfig() *UpgradeConfig {
 		BEP171Height:                      math.MaxInt64,
 		FixFailAckPackageHeight:           math.MaxInt64,
 		EnableAccountScriptsForCrossChainTransferHeight: math.MaxInt64,
+		EnableReconciliationHeight:                      math.MaxInt64,
+		DisableMessagesPhase1Height:                     math.MaxInt64,
 	}
 }
 

app/reconciliation.go

@@ -0,0 +1,142 @@
+package app
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/cosmos/cosmos-sdk/store"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	"github.com/bnb-chain/node/common"
+	"github.com/bnb-chain/node/common/types"
+)
+
+const globalAccountNumber = "globalAccountNumber"
+
+// unbalancedBlockHeightKey for saving unbalanced block height for reconciliation
+var unbalancedBlockHeightKey = []byte("0x01")
+
+// reconBalance will do reconciliation for accounts balances.
+func (app *BinanceChain) reconBalance(ctx sdk.Context) {
+	height, exists := app.getUnbalancedBlockHeight(ctx)
+	if exists {
+		panic(fmt.Sprintf("unbalanced state at block height %d, please use hardfork to bypass it", height))
+	}
+
+	accountStore, ok := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
+	if !ok {
+		panic("cannot convert account store to ival store")
+	}
+	accPre, accCurrent := app.getAccountChanges(ctx, accountStore)
+	accountStore.ResetDiff()
+
+	tokenStore, ok := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
+	if !ok {
+		panic("cannot convert token store to ival store")
+	}
+	tokenPre, tokenCurrent := app.getTokenChanges(ctx, tokenStore)
+	tokenStore.ResetDiff()
+
+	// accPre and tokenPre are positive, there will be no overflow
+	accountDiff := accCurrent.Plus(accPre.Negative())
+	tokenDiff := tokenCurrent.Plus(tokenPre.Negative())
+
+	if !accountDiff.IsEqual(tokenDiff) {
+		ctx.Logger().Error(fmt.Sprintf("unbalanced at block %d, account diff: %s, token diff: %s \n",
+			ctx.BlockHeight(), accountDiff.String(), tokenDiff.String()))
+		app.saveUnbalancedBlockHeight(ctx)
+	}
+}
+
+func (app *BinanceChain) getAccountChanges(ctx sdk.Context, accountStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
+	preCoins := sdk.Coins{}
+	currentCoins := sdk.Coins{}
+
+	diff := accountStore.GetDiff()
+	version := accountStore.GetTree().Version() - 1
+	for k, v := range diff {
+		if k == globalAccountNumber {
+			continue
+		}
+		var acc1 sdk.Account
+		err := app.Codec.UnmarshalBinaryBare(v, &acc1)
+		if err != nil {
+			panic("failed to unmarshal diff value " + err.Error())
+		}
+		nacc1 := acc1.(types.NamedAccount)
+		ctx.Logger().Debug("diff account", "address", nacc1.GetAddress(), "coins", nacc1.GetCoins().String())
+		currentCoins = currentCoins.Plus(nacc1.GetCoins())
+		currentCoins = currentCoins.Plus(nacc1.GetFrozenCoins())
+		currentCoins = currentCoins.Plus(nacc1.GetLockedCoins())
+
+		var acc2 sdk.Account
+		_, v = accountStore.GetTree().GetVersioned([]byte(k), version)
+		if v != nil { // it is not a new account
+			err = app.Codec.UnmarshalBinaryBare(v, &acc2)
+			if err != nil {
+				panic("failed to unmarshal previous value " + err.Error())
+			}
+			nacc2 := acc2.(types.NamedAccount)
+
+			ctx.Logger().Debug("pre account", "address", nacc2.GetAddress(), "coins", nacc2.GetCoins().String())
+			preCoins = preCoins.Plus(nacc2.GetCoins())
+			preCoins = preCoins.Plus(nacc2.GetFrozenCoins())
+			preCoins = preCoins.Plus(nacc2.GetLockedCoins())
+		}
+	}
+	ctx.Logger().Debug("account changes", "diff", currentCoins.String(), "previous", preCoins.String(),
+		"version", version, "height", ctx.BlockHeight())
+
+	return preCoins, currentCoins
+}
+
+func (app *BinanceChain) getTokenChanges(ctx sdk.Context, tokenStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
+	preCoins := sdk.Coins{}
+	currentCoins := sdk.Coins{}
+
+	diff := tokenStore.GetDiff()
+	version := tokenStore.GetTree().Version() - 1
+	for k, v := range diff {
+		var token1 types.IToken
+		err := app.Codec.UnmarshalBinaryBare(v, &token1)
+		if err != nil {
+			panic("failed to unmarshal diff value " + err.Error())
+		}
+		currentCoins = currentCoins.Plus(sdk.Coins{
+			sdk.NewCoin(token1.GetSymbol(), token1.GetTotalSupply().ToInt64()),
+		})
+
+		var token2 types.IToken
+		_, v = tokenStore.GetTree().GetVersioned([]byte(k), version)
+		if v != nil { // it is not a new token
+			err = app.Codec.UnmarshalBinaryBare(v, &token2)
+			if err != nil {
+				panic("failed to unmarshal previous value " + err.Error())
+			}
+			preCoins = preCoins.Plus(sdk.Coins{
+				sdk.NewCoin(token2.GetSymbol(), token2.GetTotalSupply().ToInt64()),
+			})
+		}
+	}
+	ctx.Logger().Debug("token changes", "diff", currentCoins.String(), "previous", preCoins.String(),
+		"version", version, "height", ctx.BlockHeight())
+
+	return preCoins, currentCoins
+}
+
+func (app *BinanceChain) saveUnbalancedBlockHeight(ctx sdk.Context) {
+	reconStore := app.GetCommitMultiStore().GetCommitStore(common.ReconStoreKey).(*store.IavlStore)
+	bz := make([]byte, 8)
+	binary.BigEndian.PutUint64(bz[:], uint64(ctx.BlockHeight()))
+	reconStore.Set(unbalancedBlockHeightKey, bz)
+}
+
+func (app *BinanceChain) getUnbalancedBlockHeight(ctx sdk.Context) (uint64, bool) {
+	reconStore := app.GetCommitMultiStore().GetCommitStore(common.ReconStoreKey).(*store.IavlStore)
+
+	bz := reconStore.Get(unbalancedBlockHeightKey)
+	if bz == nil {
+		return 0, false
+	}
+	return binary.BigEndian.Uint64(bz), true
+}

app/reconciliation_test.go

@@ -0,0 +1,66 @@
+package app
+
+import (
+	"github.com/stretchr/testify/require"
+	"math"
+	"testing"
+
+	"github.com/cosmos/cosmos-sdk/types"
+)
+
+func Test_Reconciliation_Overflow(t *testing.T) {
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("should panic for overflow")
+		}
+	}()
+
+	accountPre := types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+	accountCurrent := types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+	tokenPre := types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+	tokenCurrent := types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+
+	_ = accountPre.Plus(tokenCurrent).IsEqual(accountCurrent.Plus(tokenPre))
+}
+
+func Test_Reconciliation_NoOverflow(t *testing.T) {
+	accountPre := types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+	accountCurrent := types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+	tokenPre := types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+	tokenCurrent := types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+
+	equal := accountCurrent.Plus(accountPre.Negative()).IsEqual(tokenCurrent.Plus(tokenPre.Negative()))
+	require.True(t, equal)
+
+	accountPre = types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+	accountCurrent = types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+	tokenPre = types.Coins{
+		types.NewCoin("BNB", math.MaxInt64),
+	}
+	tokenCurrent = types.Coins{
+		types.NewCoin("BNB", 10),
+	}
+
+	equal = accountCurrent.Plus(accountPre.Negative()).IsEqual(tokenCurrent.Plus(tokenPre.Negative()))
+	require.True(t, equal)
+}

common/stores.go

@@ -20,6 +20,7 @@ const (
 	OracleStoreName      = "oracle"
 	IbcStoreName         = "ibc"
 	SideChainStoreName   = "sc"
+	ReconStoreName       = "recon"
 
 	StakeTransientStoreName  = "transient_stake"
 	ParamsTransientStoreName = "transient_params"
@@ -44,6 +45,7 @@ var (
 	OracleStoreKey      = sdk.NewKVStoreKey(OracleStoreName)
 	IbcStoreKey         = sdk.NewKVStoreKey(IbcStoreName)
 	SideChainStoreKey   = sdk.NewKVStoreKey(SideChainStoreName)
+	ReconStoreKey       = sdk.NewKVStoreKey(ReconStoreName)
 
 	TStakeStoreKey  = sdk.NewTransientStoreKey(StakeTransientStoreName)
 	TParamsStoreKey = sdk.NewTransientStoreKey(ParamsTransientStoreName)
@@ -66,6 +68,7 @@ var (
 		SideChainStoreName:       SideChainStoreKey,
 		BridgeStoreName:          BridgeStoreKey,
 		OracleStoreName:          OracleStoreKey,
+		ReconStoreName:           ReconStoreKey,
 		StakeTransientStoreName:  TStakeStoreKey,
 		ParamsTransientStoreName: TParamsStoreKey,
 	}
@@ -88,6 +91,7 @@ var (
 		SideChainStoreName,
 		BridgeStoreName,
 		OracleStoreName,
+		ReconStoreName,
 	}
 )
 

common/upgrade/upgrade.go

@@ -49,6 +49,8 @@ const (
 	BEP171                      = sdk.BEP171 // https://github.com/bnb-chain/BEPs/pull/171 Security Enhancement for Cross-Chain Module
 	BEP173                      = sdk.BEP173 // https://github.com/bnb-chain/BEPs/pull/173 Text Proposal
 	FixDoubleSignChainId        = sdk.FixDoubleSignChainId
+	EnableReconciliation        = sdk.EnableReconciliation
+	DisableMessagesPhase1       = sdk.DisableMessagesPhase1
 )
 
 func UpgradeBEP10(before func(), after func()) {

go.mod

@@ -88,7 +88,7 @@ require (
 )
 
 replace (
-	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/bnc-cosmos-sdk v0.26.5
+	github.com/cosmos/cosmos-sdk => github.com/forcodedancing/bnc-cosmos-sdk v0.25.8-0.20230531023724-2e22c6326b67
 	github.com/tendermint/go-amino => github.com/bnb-chain/bnc-go-amino v0.14.1-binance.2
 	github.com/tendermint/iavl => github.com/bnb-chain/bnc-tendermint-iavl v0.12.0-binance.5
 	github.com/tendermint/tendermint => github.com/bnb-chain/bnc-tendermint v0.32.3-bc.10