add pendingmatch flag

app/app.go

@@ -265,7 +265,7 @@ func SetUpgradeConfig(upgradeConfig *config.UpgradeConfig) {
 	upgrade.Mgr.AddUpgradeHeight(upgrade.LotSizeOptimization, upgradeConfig.LotSizeUpgradeHeight)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.ListingRuleUpgrade, upgradeConfig.ListingRuleUpgradeHeight)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.FixZeroBalance, upgradeConfig.FixZeroBalanceHeight)
-	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP70, upgradeConfig.BEP70Height)
+	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP67, upgradeConfig.BEP67Height)
 
 	// register store keys of upgrade
 	upgrade.Mgr.RegisterStoreKeys(upgrade.BEP9, common.TimeLockStoreKey.Name())

app/apptest/match_allocation_test.go

@@ -230,7 +230,7 @@ func GetOrderId(add sdk.AccAddress, seq int64, ctx sdk.Context) string {
 func GetOrderBook(pair string) ([]level, []level) {
 	buys := make([]level, 0)
 	sells := make([]level, 0)
-	orderbooks := testApp.DexKeeper.GetOrderBookLevels(pair, 25)
+	orderbooks, _ := testApp.DexKeeper.GetOrderBookLevels(pair, 25)
 	for _, l := range orderbooks {
 		if l.BuyPrice != 0 {
 			buys = append(buys, level{price: l.BuyPrice, qty: l.BuyQty})

app/apptest/ordertx_test.go

@@ -22,10 +22,10 @@ type level struct {
 	qty   utils.Fixed8
 }
 
-func getOrderBook(pair string) ([]level, []level) {
+func getOrderBook(pair string) ([]level, []level, bool) {
 	buys := make([]level, 0)
 	sells := make([]level, 0)
-	orderbooks := testApp.DexKeeper.GetOrderBookLevels(pair, 5)
+	orderbooks, pendingMatch := testApp.DexKeeper.GetOrderBookLevels(pair, 5)
 	for _, l := range orderbooks {
 		if l.BuyPrice != 0 {
 			buys = append(buys, level{price: l.BuyPrice, qty: l.BuyQty})
@@ -34,7 +34,7 @@ func getOrderBook(pair string) ([]level, []level) {
 			sells = append(sells, level{price: l.SellPrice, qty: l.SellQty})
 		}
 	}
-	return buys, sells
+	return buys, sells, pendingMatch
 }
 
 func genOrderID(add sdk.AccAddress, seq int64, ctx sdk.Context, am auth.AccountKeeper) string {
@@ -125,6 +125,10 @@ func Test_handleNewOrder_DeliverTx(t *testing.T) {
 	testApp.DexKeeper.PairMapper.AddTradingPair(ctx, tradingPair)
 	testApp.DexKeeper.AddEngine(tradingPair)
 
+	tradingPair2 := types.NewTradingPair("ETH-001", "BNB", 1e8)
+	testApp.DexKeeper.PairMapper.AddTradingPair(ctx, tradingPair2)
+	testApp.DexKeeper.AddEngine(tradingPair2)
+
 	add := Account(0).GetAddress()
 	oid := fmt.Sprintf("%X-0", add)
 	msg := o.NewNewOrderMsg(add, oid, 1, "BTC-000_BNB", 355e8, 1e8)
@@ -133,11 +137,17 @@ func Test_handleNewOrder_DeliverTx(t *testing.T) {
 	t.Logf("res is %v and error is %v", res, e)
 	assert.Equal(uint32(0), res.Code)
 	assert.Nil(e)
-	buys, sells := getOrderBook("BTC-000_BNB")
+	buys, sells, pendingMatch := getOrderBook("BTC-000_BNB")
 	assert.Equal(1, len(buys))
 	assert.Equal(0, len(sells))
+	assert.Equal(true, pendingMatch)
 	assert.Equal(utils.Fixed8(355e8), buys[0].price)
 	assert.Equal(utils.Fixed8(1e8), buys[0].qty)
+
+	buys, sells, pendingMatch = getOrderBook("ETH-001_BNB")
+	assert.Equal(0, len(buys))
+	assert.Equal(0, len(sells))
+	assert.Equal(false, pendingMatch)
 }
 
 func Test_Match(t *testing.T) {
@@ -196,13 +206,15 @@ func Test_Match(t *testing.T) {
 	t.Logf("res is %v and error is %v", res, e)
 	msg = o.NewNewOrderMsg(add, genOrderID(add, 3, ctx, am), 1, "BTC-000_BNB", 98e8, 300e8)
 	res, e = testClient.DeliverTxSync(msg, testApp.Codec)
-	buys, sells := getOrderBook("BTC-000_BNB")
+	buys, sells, pendingMatch := getOrderBook("BTC-000_BNB")
 	assert.Equal(4, len(buys))
 	assert.Equal(3, len(sells))
+	assert.Equal(true, pendingMatch)
 	testApp.DexKeeper.MatchAndAllocateAll(ctx, nil)
-	buys, sells = getOrderBook("BTC-000_BNB")
+	buys, sells, pendingMatch = getOrderBook("BTC-000_BNB")
 	assert.Equal(0, len(buys))
 	assert.Equal(3, len(sells))
+	assert.Equal(false, pendingMatch)
 
 	trades, lastPx := testApp.DexKeeper.GetLastTradesForPair("BTC-000_BNB")
 	assert.Equal(int64(96e8), lastPx)
@@ -247,20 +259,20 @@ func Test_Match(t *testing.T) {
 	res, e = testClient.DeliverTxSync(msg, testApp.Codec)
 	t.Logf("res is %v and error is %v", res, e)
 
-	buys, sells = getOrderBook("BTC-000_BNB")
+	buys, sells, _ = getOrderBook("BTC-000_BNB")
 	assert.Equal(0, len(buys))
 	assert.Equal(3, len(sells))
-	buys, sells = getOrderBook("ETH-000_BNB")
+	buys, sells, _ = getOrderBook("ETH-000_BNB")
 	assert.Equal(4, len(buys))
 	assert.Equal(3, len(sells))
 
 	testApp.DexKeeper.MatchAndAllocateAll(ctx, nil)
-	buys, sells = getOrderBook("ETH-000_BNB")
+	buys, sells, _ = getOrderBook("ETH-000_BNB")
 	t.Logf("buys: %v", buys)
 	t.Logf("sells: %v", sells)
 	assert.Equal(1, len(buys))
 	assert.Equal(2, len(sells))
-	buys, sells = getOrderBook("BTC-000_BNB")
+	buys, sells, _ = getOrderBook("BTC-000_BNB")
 	assert.Equal(0, len(buys))
 	assert.Equal(3, len(sells))
 	trades, lastPx = testApp.DexKeeper.GetLastTradesForPair("ETH-000_BNB")

app/config/config.go

@@ -67,8 +67,8 @@ LotSizeUpgradeHeight = {{ .UpgradeConfig.LotSizeUpgradeHeight }}
 ListingRuleUpgradeHeight = {{ .UpgradeConfig.ListingRuleUpgradeHeight }}
 # Block height of FixZeroBalanceHeight upgrade
 FixZeroBalanceHeight = {{ .UpgradeConfig.FixZeroBalanceHeight }}
-# Block height to BEP70 upgrade
-BEP70Height = {{ .UpgradeConfig.BEP70Height }}
+# Block height of BEP67 upgrade
+BEP67Height = {{ .UpgradeConfig.BEP67Height }}
 
 [query]
 # ABCI query interface black list, suggested value: ["custom/gov/proposals", "custom/timelock/timelocks", "custom/atomicSwap/swapcreator", "custom/atomicSwap/swaprecipient"]
@@ -374,6 +374,8 @@ type UpgradeConfig struct {
 	LotSizeUpgradeHeight       int64 `mapstructure:"LotSizeUpgradeHeight"`
 	ListingRuleUpgradeHeight   int64 `mapstructure:"ListingRuleUpgradeHeight"`
 	FixZeroBalanceHeight       int64 `mapstructure:"FixZeroBalanceHeight"`
+
+	BEP67Height int64 `mapstructure:"BEP67Height"`
 }
 
 func defaultUpgradeConfig() *UpgradeConfig {
@@ -390,6 +392,7 @@ func defaultUpgradeConfig() *UpgradeConfig {
 		LotSizeUpgradeHeight:       math.MaxInt64,
 		ListingRuleUpgradeHeight:   math.MaxInt64,
 		FixZeroBalanceHeight:       math.MaxInt64,
+		BEP67Height:                1,
 	}
 }
 

common/upgrade/upgrade.go

@@ -25,6 +25,8 @@ const (
 	LotSizeOptimization  = "LotSizeOptimization"
 	ListingRuleUpgrade   = "ListingRuleUpgrade" // Remove restriction that only the owner of base asset can list trading pair
 	FixZeroBalance       = "FixZeroBalance"
+
+	BEP67 = "BEP67" // https://github.com/binance-chain/BEPs/pull/67
 )
 
 func UpgradeBEP10(before func(), after func()) {

plugins/dex/abci.go

@@ -106,10 +106,11 @@ func createAbciQueryHandler(keeper *DexKeeper) app.AbciQueryHandler {
 					levelLimit = l
 				}
 			}
-			levels := keeper.GetOrderBookLevels(pair, levelLimit)
+			levels, pendingMatch := keeper.GetOrderBookLevels(pair, levelLimit)
 			book := store.OrderBook{
-				Height: height,
-				Levels: levels,
+				Height:       height,
+				Levels:       levels,
+				PendingMatch: pendingMatch,
 			}
 			bz, err := app.GetCodec().MarshalBinaryLengthPrefixed(book)
 			if err != nil {

plugins/dex/client/rest/utils/streamer.go

@@ -78,6 +78,10 @@ func StreamDepthResponse(w io.Writer, ob *store.OrderBook, limit int) error {
 		i++
 	}
 
-	// end streamed json with height
-	return write(w, fmt.Sprintf("],\"height\":%d}", ob.Height))
+	// pass 3 - height
+	if err := write(w, fmt.Sprintf("],\"height\":%d", ob.Height)); err != nil {
+		return err
+	}
+	// end streamed json with pendingMatch flag
+	return write(w, fmt.Sprintf(",\"pendingMatch\":%t}", ob.PendingMatch))
 }

plugins/dex/client/rest/utils/streamer_test.go

@@ -19,9 +19,10 @@ func TestStreamDepthResponse(t *testing.T) {
 		limit int
 	}
 	type response struct {
-		Height int64      `json:"height"`
-		Asks   [][]string `json:"asks"`
-		Bids   [][]string `json:"bids"`
+		Height       int64      `json:"height"`
+		Asks         [][]string `json:"asks"`
+		Bids         [][]string `json:"bids"`
+		PendingMatch bool       `json:"pendingMatch"`
 	}
 	tests := []struct {
 		name    string
@@ -41,9 +42,10 @@ func TestStreamDepthResponse(t *testing.T) {
 			},
 			wantW: (func() string {
 				want, _ := json.Marshal(response{
-					Height: 1,
-					Asks:   [][]string{},
-					Bids:   [][]string{},
+					Height:       1,
+					Asks:         [][]string{},
+					Bids:         [][]string{},
+					PendingMatch: false,
 				})
 				sorted := types.MustSortJSON(want)
 				return string(sorted)
@@ -82,6 +84,7 @@ func TestStreamDepthResponse(t *testing.T) {
 					Bids: [][]string{
 						{"100.00000000", "1.00000000"},
 					},
+					PendingMatch: false,
 				})
 				sorted := types.MustSortJSON(want)
 				return string(sorted)
@@ -126,6 +129,7 @@ func TestStreamDepthResponse(t *testing.T) {
 						{"150.00000000", "1.00000000"},
 						{"100.00000000", "1.00000000"},
 					},
+					PendingMatch: false,
 				})
 				sorted := types.MustSortJSON(want)
 				return string(sorted)
@@ -165,6 +169,7 @@ func TestStreamDepthResponse(t *testing.T) {
 						// highest buy first
 						{"100.00000000", "1.00000000"},
 					},
+					PendingMatch: false,
 				})
 				sorted := types.MustSortJSON(want)
 				return string(sorted)

plugins/dex/matcheng/orderbook.go

@@ -20,7 +20,8 @@ type OrderBookInterface interface {
 	GetOrder(id string, side int8, price int64) (OrderPart, error)
 	RemoveOrder(id string, side int8, price int64) (OrderPart, error)
 	RemoveOrders(beforeTime int64, side int8, cb func(OrderPart)) error
-	UpdateForEachPriceLevel(side int8, updater func(*PriceLevel))
+	RemoveOrdersBasedOnPriceLevel(expireTime int64, forceExpireTime int64, priceLevelsToReserve int, side int8, removeCallback func(ord OrderPart)) error
+	UpdateForEachPriceLevel(side int8, updater LevelIter)
 	GetPriceLevel(price int64, side int8) *PriceLevel
 	RemovePriceLevel(price int64, side int8) int
 	ShowDepth(maxLevels int, iterBuy LevelIter, iterSell LevelIter)
@@ -130,14 +131,26 @@ func (ob *OrderBookOnULList) RemoveOrder(id string, side int8, price int64) (Ord
 }
 
 func (ob *OrderBookOnULList) RemoveOrders(beforeTime int64, side int8, cb func(OrderPart)) error {
-	ob.UpdateForEachPriceLevel(side, func(pl *PriceLevel) {
+	ob.UpdateForEachPriceLevel(side, func(pl *PriceLevel, levelIndex int) {
 		pl.removeOrders(beforeTime, cb)
 	})
 
 	return nil
 }
 
-func (ob *OrderBookOnULList) UpdateForEachPriceLevel(side int8, updater func(*PriceLevel)) {
+// order beyond priceLevelsToReserve will be expired if it's placed before expireTime. All orders will be expired if they are placed before forceExpireTime
+func (ob *OrderBookOnULList) RemoveOrdersBasedOnPriceLevel(expireTime int64, forceExpireTime int64, priceLevelsToReserve int, side int8, removeCallback func(ord OrderPart)) error {
+	ob.UpdateForEachPriceLevel(side, func(pl *PriceLevel, levelIndex int) {
+		if levelIndex < priceLevelsToReserve {
+			pl.removeOrders(forceExpireTime, removeCallback)
+		} else {
+			pl.removeOrders(expireTime, removeCallback)
+		}
+	})
+	return nil
+}
+
+func (ob *OrderBookOnULList) UpdateForEachPriceLevel(side int8, updater LevelIter) {
 	q := ob.getSideQueue(side)
 	q.UpdateForEach(updater)
 }
@@ -174,11 +187,11 @@ func (ob *OrderBookOnULList) GetAllLevels() ([]PriceLevel, []PriceLevel) {
 	buys := make([]PriceLevel, 0, ob.buyQueue.capacity)
 	sells := make([]PriceLevel, 0, ob.sellQueue.capacity)
 	ob.buyQueue.Iterate(ob.buyQueue.capacity,
-		func(p *PriceLevel) {
+		func(p *PriceLevel, levelIndex int) {
 			buys = append(buys, *p)
 		})
 	ob.sellQueue.Iterate(ob.sellQueue.capacity,
-		func(p *PriceLevel) {
+		func(p *PriceLevel, levelIndex int) {
 			sells = append(sells, *p)
 		})
 	return buys, sells

plugins/dex/matcheng/orderbook_test.go

@@ -559,6 +559,32 @@ func TestOrderBookOnULList_RemoveOrders(t *testing.T) {
 	require.Len(t, sells, 0)
 }
 
+func TestOrderBookOnULList_RemoveOrdersBiasedly(t *testing.T) {
+	book := NewOrderBookOnULList(16, 4)
+	book.InsertOrder("1", BUYSIDE, 10000, 1000, 10000)
+	book.InsertOrder("2", BUYSIDE, 10001, 1000, 10000)
+	book.InsertOrder("3", BUYSIDE, 10002, 1000, 10000)
+	err := book.RemoveOrdersBasedOnPriceLevel(9999, 10001, 10, BUYSIDE, nil)
+	require.NoError(t, err)
+	buys, sells := book.GetAllLevels()
+	//t.Log(buys)
+	//t.Log(sells)
+	require.Len(t, buys, 1)
+	require.Len(t, buys[0].Orders, 2)
+	require.Len(t, sells, 0)
+	err = book.RemoveOrdersBasedOnPriceLevel(10003, 10001, 10, BUYSIDE, nil)
+	require.NoError(t, err)
+	buys, sells = book.GetAllLevels()
+	require.Len(t, buys, 1)
+	require.Len(t, buys[0].Orders, 2)
+	require.Len(t, sells, 0)
+	err = book.RemoveOrdersBasedOnPriceLevel(10003, 10003, 10, BUYSIDE, nil)
+	require.NoError(t, err)
+	buys, sells = book.GetAllLevels()
+	require.Len(t, buys, 0)
+	require.Len(t, sells, 0)
+}
+
 func TestOrderBookOnULList_GetOverlappedRange(t *testing.T) {
 	overlap := make([]OverLappedLevel, 4)
 	buyBuf := make([]PriceLevel, 16)

plugins/dex/matcheng/types.go

@@ -190,7 +190,7 @@ func (overlapped *OverLappedLevel) HasSellTaker() bool {
 	return overlapped.SellTakerStartIdx < len(overlapped.SellOrders)
 }
 
-type LevelIter func(*PriceLevel)
+type LevelIter func(priceLevel *PriceLevel, levelIndex int)
 
 type MergedPriceLevel struct {
 	price    int64

plugins/dex/matcheng/unrolledlinkedlist.go

@@ -331,7 +331,7 @@ func (ull *ULList) Iterate(levelNum int, iter LevelIter) {
 			if curLevel >= levelNum {
 				return
 			}
-			iter(&b.elements[i])
+			iter(&b.elements[i], curLevel)
 			curLevel += 1
 		}
 	}
@@ -394,6 +394,7 @@ func (ull *ULList) GetPriceLevel(p float64) *PriceLevel {
 func (ull *ULList) UpdateForEach(updater LevelIter) {
 	b := ull.begin
 	var last *bucket
+	levelIndex := 0
 	for b != ull.dend {
 		for i := 0; ; {
 			k := len(b.elements)
@@ -403,7 +404,8 @@ func (ull *ULList) UpdateForEach(updater LevelIter) {
 				break
 			}
 			pl := &b.elements[i]
-			updater(pl)
+			updater(pl, levelIndex)
+			levelIndex++
 			if len(pl.Orders) == 0 {
 				b.deleteElement(i)
 			} else {