From 909fc3a453e2cb0614f048851694841d05db0325 Mon Sep 17 00:00:00 2001 From: jayy04 <103467857+jayy04@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:15:03 -0400 Subject: [PATCH] paginate liquidation daemon response (#2118) --- protocol/daemons/flags/flags.go | 29 +++- protocol/daemons/flags/flags_test.go | 3 + .../daemons/liquidation/client/grpc_helper.go | 143 +++++++++++++++++- .../liquidation/client/grpc_helper_test.go | 52 +++++-- .../liquidation/client/sub_task_runner.go | 1 + .../client/sub_task_runner_test.go | 85 ++++++++--- protocol/daemons/server/liquidation.go | 10 +- .../liquidations/daemon_liquidation_info.go | 112 +++++++++----- .../daemon_liquidation_info_test.go | 50 +++--- protocol/x/clob/abci_test.go | 5 +- protocol/x/clob/e2e/withdrawal_gating_test.go | 3 + protocol/x/clob/keeper/deleveraging_test.go | 2 +- protocol/x/clob/keeper/liquidations_test.go | 1 + 13 files changed, 389 insertions(+), 107 deletions(-) diff --git a/protocol/daemons/flags/flags.go b/protocol/daemons/flags/flags.go index ee7eeeceac..939e32f68d 100644 --- a/protocol/daemons/flags/flags.go +++ b/protocol/daemons/flags/flags.go @@ -1,11 +1,12 @@ package flags import ( + "time" + servertypes "github.com/cosmos/cosmos-sdk/server/types" oracleconfig "github.com/skip-mev/slinky/oracle/config" "github.com/spf13/cast" "github.com/spf13/cobra" - "time" ) // List of CLI flags for Server and Client. @@ -22,9 +23,10 @@ const ( FlagBridgeDaemonLoopDelayMs = "bridge-daemon-loop-delay-ms" FlagBridgeDaemonEthRpcEndpoint = "bridge-daemon-eth-rpc-endpoint" - FlagLiquidationDaemonEnabled = "liquidation-daemon-enabled" - FlagLiquidationDaemonLoopDelayMs = "liquidation-daemon-loop-delay-ms" - FlagLiquidationDaemonQueryPageLimit = "liquidation-daemon-query-page-limit" + FlagLiquidationDaemonEnabled = "liquidation-daemon-enabled" + FlagLiquidationDaemonLoopDelayMs = "liquidation-daemon-loop-delay-ms" + FlagLiquidationDaemonQueryPageLimit = "liquidation-daemon-query-page-limit" + FlagLiquidationDaemonResponsePageLimit = "liquidation-daemon-response-page-limit" // Oracle flags FlagOracleEnabled = "oracle.enabled" @@ -62,6 +64,8 @@ type LiquidationFlags struct { LoopDelayMs uint32 // QueryPageLimit configures the pagination limit for fetching subaccounts. QueryPageLimit uint64 + // ResponsePageLimit configures the pagination limit for the response to application. + ResponsePageLimit uint64 } // PriceFlags contains configuration flags for the Price Daemon. @@ -102,9 +106,10 @@ func GetDefaultDaemonFlags() DaemonFlags { EthRpcEndpoint: "", }, Liquidation: LiquidationFlags{ - Enabled: true, - LoopDelayMs: 1_600, - QueryPageLimit: 1_000, + Enabled: true, + LoopDelayMs: 1_600, + QueryPageLimit: 1_000, + ResponsePageLimit: 2_000, }, Price: PriceFlags{ Enabled: false, @@ -183,6 +188,11 @@ func AddDaemonFlagsToCmd( df.Liquidation.QueryPageLimit, "Limit on the number of items to fetch per query in the Liquidation Daemon task loop.", ) + cmd.Flags().Uint64( + FlagLiquidationDaemonResponsePageLimit, + df.Liquidation.ResponsePageLimit, + "Limit on the number of items to send to the main application in the Liquidation Daemon task loop.", + ) // Price Daemon. cmd.Flags().Bool( @@ -276,6 +286,11 @@ func GetDaemonFlagValuesFromOptions( result.Liquidation.QueryPageLimit = v } } + if option := appOpts.Get(FlagLiquidationDaemonResponsePageLimit); option != nil { + if v, err := cast.ToUint64E(option); err == nil { + result.Liquidation.ResponsePageLimit = v + } + } // Price Daemon. if option := appOpts.Get(FlagPriceDaemonEnabled); option != nil { diff --git a/protocol/daemons/flags/flags_test.go b/protocol/daemons/flags/flags_test.go index e94a055d45..169acef479 100644 --- a/protocol/daemons/flags/flags_test.go +++ b/protocol/daemons/flags/flags_test.go @@ -26,6 +26,7 @@ func TestAddDaemonFlagsToCmd(t *testing.T) { flags.FlagLiquidationDaemonEnabled, flags.FlagLiquidationDaemonLoopDelayMs, flags.FlagLiquidationDaemonQueryPageLimit, + flags.FlagLiquidationDaemonResponsePageLimit, flags.FlagPriceDaemonEnabled, flags.FlagPriceDaemonLoopDelayMs, @@ -53,6 +54,7 @@ func TestGetDaemonFlagValuesFromOptions_Custom(t *testing.T) { optsMap[flags.FlagLiquidationDaemonEnabled] = true optsMap[flags.FlagLiquidationDaemonLoopDelayMs] = uint32(2222) optsMap[flags.FlagLiquidationDaemonQueryPageLimit] = uint64(3333) + optsMap[flags.FlagLiquidationDaemonResponsePageLimit] = uint64(4444) optsMap[flags.FlagPriceDaemonEnabled] = true optsMap[flags.FlagPriceDaemonLoopDelayMs] = uint32(4444) @@ -83,6 +85,7 @@ func TestGetDaemonFlagValuesFromOptions_Custom(t *testing.T) { require.Equal(t, optsMap[flags.FlagLiquidationDaemonEnabled], r.Liquidation.Enabled) require.Equal(t, optsMap[flags.FlagLiquidationDaemonLoopDelayMs], r.Liquidation.LoopDelayMs) require.Equal(t, optsMap[flags.FlagLiquidationDaemonQueryPageLimit], r.Liquidation.QueryPageLimit) + require.Equal(t, optsMap[flags.FlagLiquidationDaemonResponsePageLimit], r.Liquidation.ResponsePageLimit) // Price Daemon. require.Equal(t, optsMap[flags.FlagPriceDaemonEnabled], r.Price.Enabled) diff --git a/protocol/daemons/liquidation/client/grpc_helper.go b/protocol/daemons/liquidation/client/grpc_helper.go index aa662f0cc8..691c467a01 100644 --- a/protocol/daemons/liquidation/client/grpc_helper.go +++ b/protocol/daemons/liquidation/client/grpc_helper.go @@ -211,6 +211,7 @@ func (c *Client) SendLiquidatableSubaccountIds( liquidatableSubaccountIds []satypes.SubaccountId, negativeTncSubaccountIds []satypes.SubaccountId, openPositionInfoMap map[uint32]*clobtypes.SubaccountOpenPositionInfo, + pageLimit uint64, ) error { defer telemetry.ModuleMeasureSince( metrics.LiquidationDaemon, @@ -241,19 +242,145 @@ func (c *Client) SendLiquidatableSubaccountIds( subaccountOpenPositionInfo = append(subaccountOpenPositionInfo, *openPositionInfoMap[perpetualId]) } - request := &api.LiquidateSubaccountsRequest{ - BlockHeight: blockHeight, - LiquidatableSubaccountIds: liquidatableSubaccountIds, - NegativeTncSubaccountIds: negativeTncSubaccountIds, - SubaccountOpenPositionInfo: subaccountOpenPositionInfo, - } + // Break this down to multiple requests if the number of subaccounts is too large. + + // Liquidatable subaccount ids. + requests := GenerateLiquidateSubaccountsPaginatedRequests( + liquidatableSubaccountIds, + blockHeight, + pageLimit, + ) - if _, err := c.LiquidationServiceClient.LiquidateSubaccounts(ctx, request); err != nil { - return err + // Negative TNC subaccount ids. + requests = append( + requests, + GenerateNegativeTNCSubaccountsPaginatedRequests( + negativeTncSubaccountIds, + blockHeight, + pageLimit, + )..., + ) + + // Subaccount open position info. + requests = append( + requests, + GenerateSubaccountOpenPositionPaginatedRequests( + subaccountOpenPositionInfo, + blockHeight, + pageLimit, + )..., + ) + + for _, req := range requests { + if _, err := c.LiquidationServiceClient.LiquidateSubaccounts(ctx, req); err != nil { + return err + } } + return nil } +func GenerateLiquidateSubaccountsPaginatedRequests( + ids []satypes.SubaccountId, + blockHeight uint32, + pageLimit uint64, +) []*api.LiquidateSubaccountsRequest { + if len(ids) == 0 { + return []*api.LiquidateSubaccountsRequest{ + { + BlockHeight: blockHeight, + LiquidatableSubaccountIds: []satypes.SubaccountId{}, + }, + } + } + + requests := make([]*api.LiquidateSubaccountsRequest, 0) + for start := 0; start < len(ids); start += int(pageLimit) { + end := lib.Min(start+int(pageLimit), len(ids)) + request := &api.LiquidateSubaccountsRequest{ + BlockHeight: blockHeight, + LiquidatableSubaccountIds: ids[start:end], + } + requests = append(requests, request) + } + return requests +} + +func GenerateNegativeTNCSubaccountsPaginatedRequests( + ids []satypes.SubaccountId, + blockHeight uint32, + pageLimit uint64, +) []*api.LiquidateSubaccountsRequest { + if len(ids) == 0 { + return []*api.LiquidateSubaccountsRequest{ + { + BlockHeight: blockHeight, + NegativeTncSubaccountIds: []satypes.SubaccountId{}, + }, + } + } + + requests := make([]*api.LiquidateSubaccountsRequest, 0) + for start := 0; start < len(ids); start += int(pageLimit) { + end := lib.Min(start+int(pageLimit), len(ids)) + request := &api.LiquidateSubaccountsRequest{ + BlockHeight: blockHeight, + NegativeTncSubaccountIds: ids[start:end], + } + requests = append(requests, request) + } + return requests +} + +func GenerateSubaccountOpenPositionPaginatedRequests( + subaccountOpenPositionInfo []clobtypes.SubaccountOpenPositionInfo, + blockHeight uint32, + pageLimit uint64, +) []*api.LiquidateSubaccountsRequest { + if len(subaccountOpenPositionInfo) == 0 { + return []*api.LiquidateSubaccountsRequest{ + { + BlockHeight: blockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{}, + }, + } + } + + requests := make([]*api.LiquidateSubaccountsRequest, 0) + for _, info := range subaccountOpenPositionInfo { + // Long positions. + for start := 0; start < len(info.SubaccountsWithLongPosition); start += int(pageLimit) { + end := lib.Min(start+int(pageLimit), len(info.SubaccountsWithLongPosition)) + request := &api.LiquidateSubaccountsRequest{ + BlockHeight: blockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ + { + PerpetualId: info.PerpetualId, + SubaccountsWithLongPosition: info.SubaccountsWithLongPosition[start:end], + }, + }, + } + requests = append(requests, request) + } + + // Short positions. + for start := 0; start < len(info.SubaccountsWithShortPosition); start += int(pageLimit) { + end := lib.Min(start+int(pageLimit), len(info.SubaccountsWithShortPosition)) + request := &api.LiquidateSubaccountsRequest{ + BlockHeight: blockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ + { + PerpetualId: info.PerpetualId, + SubaccountsWithShortPosition: info.SubaccountsWithShortPosition[start:end], + }, + }, + } + requests = append(requests, request) + } + } + return requests +} + func newContextWithQueryBlockHeight( ctx context.Context, blockHeight uint32, diff --git a/protocol/daemons/liquidation/client/grpc_helper_test.go b/protocol/daemons/liquidation/client/grpc_helper_test.go index 1b25ff48b8..73d1ad5a72 100644 --- a/protocol/daemons/liquidation/client/grpc_helper_test.go +++ b/protocol/daemons/liquidation/client/grpc_helper_test.go @@ -469,7 +469,19 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { req := &api.LiquidateSubaccountsRequest{ BlockHeight: uint32(50), LiquidatableSubaccountIds: []satypes.SubaccountId{constants.Alice_Num0, constants.Bob_Num0}, - NegativeTncSubaccountIds: []satypes.SubaccountId{constants.Carl_Num0, constants.Dave_Num0}, + } + response := &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: uint32(50), + NegativeTncSubaccountIds: []satypes.SubaccountId{constants.Carl_Num0, constants.Dave_Num0}, + } + response = &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: uint32(50), SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ { PerpetualId: 0, @@ -477,6 +489,17 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { constants.Alice_Num0, constants.Carl_Num0, }, + }, + }, + } + response = &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: uint32(50), + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ + { + PerpetualId: 0, SubaccountsWithShortPosition: []satypes.SubaccountId{ constants.Bob_Num0, constants.Dave_Num0, @@ -484,7 +507,7 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { }, }, } - response := &api.LiquidateSubaccountsResponse{} + response = &api.LiquidateSubaccountsResponse{} mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) }, liquidatableSubaccountIds: []satypes.SubaccountId{ @@ -512,12 +535,24 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { "Success Empty": { setupMocks: func(ctx context.Context, mck *mocks.QueryClient) { req := &api.LiquidateSubaccountsRequest{ + BlockHeight: uint32(50), + LiquidatableSubaccountIds: []satypes.SubaccountId{}, + } + response := &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: uint32(50), + NegativeTncSubaccountIds: []satypes.SubaccountId{}, + } + response = &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ BlockHeight: uint32(50), - LiquidatableSubaccountIds: []satypes.SubaccountId{}, - NegativeTncSubaccountIds: []satypes.SubaccountId{}, SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{}, } - response := &api.LiquidateSubaccountsResponse{} + response = &api.LiquidateSubaccountsResponse{} mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) }, liquidatableSubaccountIds: []satypes.SubaccountId{}, @@ -527,10 +562,8 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { "Errors are propagated": { setupMocks: func(ctx context.Context, mck *mocks.QueryClient) { req := &api.LiquidateSubaccountsRequest{ - BlockHeight: uint32(50), - LiquidatableSubaccountIds: []satypes.SubaccountId{}, - NegativeTncSubaccountIds: []satypes.SubaccountId{}, - SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{}, + BlockHeight: uint32(50), + LiquidatableSubaccountIds: []satypes.SubaccountId{}, } mck.On("LiquidateSubaccounts", ctx, req).Return(nil, errors.New("test error")) }, @@ -555,6 +588,7 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) { tc.liquidatableSubaccountIds, tc.negativeTncSubaccountIds, tc.subaccountOpenPositionInfo, + 1000, ) require.Equal(t, tc.expectedError, err) }) diff --git a/protocol/daemons/liquidation/client/sub_task_runner.go b/protocol/daemons/liquidation/client/sub_task_runner.go index 506d013a9c..9d5a0c37e7 100644 --- a/protocol/daemons/liquidation/client/sub_task_runner.go +++ b/protocol/daemons/liquidation/client/sub_task_runner.go @@ -83,6 +83,7 @@ func (s *SubTaskRunnerImpl) RunLiquidationDaemonTaskLoop( liquidatableSubaccountIds, negativeTncSubaccountIds, subaccountOpenPositionInfo, + liqFlags.ResponsePageLimit, ) if err != nil { return err diff --git a/protocol/daemons/liquidation/client/sub_task_runner_test.go b/protocol/daemons/liquidation/client/sub_task_runner_test.go index 727accfae4..f140a4bc79 100644 --- a/protocol/daemons/liquidation/client/sub_task_runner_test.go +++ b/protocol/daemons/liquidation/client/sub_task_runner_test.go @@ -88,8 +88,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Can get liquidatable subaccount with long position": { @@ -148,8 +147,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Skip well collateralized subaccounts": { @@ -209,8 +207,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Skip subaccounts with no open positions": { @@ -259,8 +256,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { NegativeTncSubaccountIds: []satypes.SubaccountId{}, SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{}, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Can get subaccount that become undercollateralized with funding payments (short)": { @@ -340,8 +336,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Can get subaccount that become liquidatable with funding payments (long)": { @@ -421,8 +416,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Skips subaccount that become well-collateralized with funding payments (short)": { @@ -500,8 +494,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Skips subaccount that become well-collateralized with funding payments (long)": { @@ -579,8 +572,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Can get negative tnc subaccount with short position": { @@ -642,8 +634,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, "Can get negative tnc subaccount with long position": { @@ -705,8 +696,7 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }, }, } - response3 := &api.LiquidateSubaccountsResponse{} - mck.On("LiquidateSubaccounts", ctx, req).Return(response3, nil) + setupMockLiquidateSubaccountRequests(mck, ctx, req) }, }, } @@ -739,3 +729,58 @@ func TestRunLiquidationDaemonTaskLoop(t *testing.T) { }) } } + +func setupMockLiquidateSubaccountRequests( + mck *mocks.QueryClient, + ctx context.Context, + request *api.LiquidateSubaccountsRequest, +) { + req := &api.LiquidateSubaccountsRequest{ + BlockHeight: request.BlockHeight, + LiquidatableSubaccountIds: request.LiquidatableSubaccountIds, + } + response := &api.LiquidateSubaccountsResponse{} + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: request.BlockHeight, + NegativeTncSubaccountIds: request.NegativeTncSubaccountIds, + } + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + + if len(request.SubaccountOpenPositionInfo) == 0 { + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: request.BlockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{}, + } + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + } else { + for _, info := range request.SubaccountOpenPositionInfo { + if len(info.SubaccountsWithLongPosition) > 0 { + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: request.BlockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ + { + PerpetualId: info.PerpetualId, + SubaccountsWithLongPosition: info.SubaccountsWithLongPosition, + }, + }, + } + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + } + + if len(info.SubaccountsWithShortPosition) > 0 { + req = &api.LiquidateSubaccountsRequest{ + BlockHeight: request.BlockHeight, + SubaccountOpenPositionInfo: []clobtypes.SubaccountOpenPositionInfo{ + { + PerpetualId: info.PerpetualId, + SubaccountsWithShortPosition: info.SubaccountsWithShortPosition, + }, + }, + } + mck.On("LiquidateSubaccounts", ctx, req).Return(response, nil) + } + } + } +} diff --git a/protocol/daemons/server/liquidation.go b/protocol/daemons/server/liquidation.go index 573ede7eb7..aa33424ac8 100644 --- a/protocol/daemons/server/liquidation.go +++ b/protocol/daemons/server/liquidation.go @@ -42,10 +42,12 @@ func (s *Server) LiquidateSubaccounts( metrics.Count, ) - s.daemonLiquidationInfo.UpdateBlockHeight(req.BlockHeight) - s.daemonLiquidationInfo.UpdateLiquidatableSubaccountIds(req.LiquidatableSubaccountIds) - s.daemonLiquidationInfo.UpdateNegativeTncSubaccountIds(req.NegativeTncSubaccountIds) - s.daemonLiquidationInfo.UpdateSubaccountsWithPositions(req.SubaccountOpenPositionInfo) + s.daemonLiquidationInfo.Update( + req.BlockHeight, + req.LiquidatableSubaccountIds, + req.NegativeTncSubaccountIds, + req.SubaccountOpenPositionInfo, + ) // Capture valid responses in metrics. s.reportValidResponse(types.LiquidationsDaemonServiceName) diff --git a/protocol/daemons/server/types/liquidations/daemon_liquidation_info.go b/protocol/daemons/server/types/liquidations/daemon_liquidation_info.go index dea31851d3..57c78ed515 100644 --- a/protocol/daemons/server/types/liquidations/daemon_liquidation_info.go +++ b/protocol/daemons/server/types/liquidations/daemon_liquidation_info.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "sync" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" @@ -26,27 +27,88 @@ func NewDaemonLiquidationInfo() *DaemonLiquidationInfo { } } -// UpdateBlockHeight updates the struct with the given block height. -func (ls *DaemonLiquidationInfo) UpdateBlockHeight(blockHeight uint32) { +func (ls *DaemonLiquidationInfo) Update( + blockHeight uint32, + liquidatableSubaccountIds []satypes.SubaccountId, + negativeTncSubaccountIds []satypes.SubaccountId, + subaccountsWithPositions []clobtypes.SubaccountOpenPositionInfo, +) { ls.Lock() defer ls.Unlock() - ls.blockHeight = blockHeight + + if blockHeight > ls.blockHeight { + ls.liquidatableSubaccountIds = make([]satypes.SubaccountId, 0) + ls.negativeTncSubaccountIds = make([]satypes.SubaccountId, 0) + ls.subaccountsWithPositions = make(map[uint32]*clobtypes.SubaccountOpenPositionInfo) + } else if blockHeight < ls.blockHeight { + panic( + fmt.Sprintf( + "UpdateLiquidatableSubaccountIds: block height %d cannot be less than the current block height %d", + blockHeight, + ls.blockHeight, + ), + ) + } + ls.UpdateBlockHeight(blockHeight) + + ls.UpdateLiquidatableSubaccountIds(liquidatableSubaccountIds, blockHeight) + ls.UpdateNegativeTncSubaccountIds(negativeTncSubaccountIds, blockHeight) + ls.UpdateSubaccountsWithPositions(subaccountsWithPositions, blockHeight) } -// GetBlockHeight returns the block height of the last update. -func (ls *DaemonLiquidationInfo) GetBlockHeight() uint32 { - ls.Lock() - defer ls.Unlock() - return ls.blockHeight +// UpdateBlockHeight updates the struct with the given block height. +func (ls *DaemonLiquidationInfo) UpdateBlockHeight(blockHeight uint32) { + ls.blockHeight = blockHeight } // UpdateLiquidatableSubaccountIds updates the struct with the given a list of potentially // liquidatable subaccount ids. -func (ls *DaemonLiquidationInfo) UpdateLiquidatableSubaccountIds(updates []satypes.SubaccountId) { +func (ls *DaemonLiquidationInfo) UpdateLiquidatableSubaccountIds( + updates []satypes.SubaccountId, + blockHeight uint32, +) { + ls.liquidatableSubaccountIds = append(ls.liquidatableSubaccountIds, updates...) +} + +// UpdateNegativeTncSubaccountIds updates the struct with the given a list of subaccount ids +// with negative total net collateral. +func (ls *DaemonLiquidationInfo) UpdateNegativeTncSubaccountIds( + updates []satypes.SubaccountId, + blockHeight uint32, +) { + ls.negativeTncSubaccountIds = append(ls.negativeTncSubaccountIds, updates...) +} + +// UpdateSubaccountsWithPositions updates the struct with the given a list of subaccount ids with open positions. +func (ls *DaemonLiquidationInfo) UpdateSubaccountsWithPositions( + subaccountsWithPositions []clobtypes.SubaccountOpenPositionInfo, + blockHeight uint32, +) { + // Append to the current map if the block height not changed. + for _, info := range subaccountsWithPositions { + if _, ok := ls.subaccountsWithPositions[info.PerpetualId]; !ok { + ls.subaccountsWithPositions[info.PerpetualId] = &clobtypes.SubaccountOpenPositionInfo{ + PerpetualId: info.PerpetualId, + SubaccountsWithLongPosition: make([]satypes.SubaccountId, 0), + SubaccountsWithShortPosition: make([]satypes.SubaccountId, 0), + } + } + ls.subaccountsWithPositions[info.PerpetualId].SubaccountsWithLongPosition = append( + ls.subaccountsWithPositions[info.PerpetualId].SubaccountsWithLongPosition, + info.SubaccountsWithLongPosition..., + ) + ls.subaccountsWithPositions[info.PerpetualId].SubaccountsWithShortPosition = append( + ls.subaccountsWithPositions[info.PerpetualId].SubaccountsWithShortPosition, + info.SubaccountsWithShortPosition..., + ) + } +} + +// GetBlockHeight returns the block height of the last update. +func (ls *DaemonLiquidationInfo) GetBlockHeight() uint32 { ls.Lock() defer ls.Unlock() - ls.liquidatableSubaccountIds = make([]satypes.SubaccountId, len(updates)) - copy(ls.liquidatableSubaccountIds, updates) + return ls.blockHeight } // GetLiquidatableSubaccountIds returns the list of potentially liquidatable subaccount ids @@ -59,15 +121,6 @@ func (ls *DaemonLiquidationInfo) GetLiquidatableSubaccountIds() []satypes.Subacc return results } -// UpdateNegativeTncSubaccountIds updates the struct with the given a list of subaccount ids -// with negative total net collateral. -func (ls *DaemonLiquidationInfo) UpdateNegativeTncSubaccountIds(updates []satypes.SubaccountId) { - ls.Lock() - defer ls.Unlock() - ls.negativeTncSubaccountIds = make([]satypes.SubaccountId, len(updates)) - copy(ls.negativeTncSubaccountIds, updates) -} - // GetNegativeTncSubaccountIds returns the list of subaccount ids with negative total net collateral // reported by the liquidation daemon. func (ls *DaemonLiquidationInfo) GetNegativeTncSubaccountIds() []satypes.SubaccountId { @@ -78,25 +131,6 @@ func (ls *DaemonLiquidationInfo) GetNegativeTncSubaccountIds() []satypes.Subacco return results } -// UpdateSubaccountsWithPositions updates the struct with the given a list of subaccount ids with open positions. -func (ls *DaemonLiquidationInfo) UpdateSubaccountsWithPositions( - subaccountsWithPositions []clobtypes.SubaccountOpenPositionInfo, -) { - ls.Lock() - defer ls.Unlock() - ls.subaccountsWithPositions = make(map[uint32]*clobtypes.SubaccountOpenPositionInfo) - for _, info := range subaccountsWithPositions { - clone := &clobtypes.SubaccountOpenPositionInfo{ - PerpetualId: info.PerpetualId, - SubaccountsWithLongPosition: make([]satypes.SubaccountId, len(info.SubaccountsWithLongPosition)), - SubaccountsWithShortPosition: make([]satypes.SubaccountId, len(info.SubaccountsWithShortPosition)), - } - copy(clone.SubaccountsWithLongPosition, info.SubaccountsWithLongPosition) - copy(clone.SubaccountsWithShortPosition, info.SubaccountsWithShortPosition) - ls.subaccountsWithPositions[info.PerpetualId] = clone - } -} - // GetSubaccountsWithOpenPositions returns the list of subaccount ids with open positions for a perpetual. func (ls *DaemonLiquidationInfo) GetSubaccountsWithOpenPositions( perpetualId uint32, diff --git a/protocol/daemons/server/types/liquidations/daemon_liquidation_info_test.go b/protocol/daemons/server/types/liquidations/daemon_liquidation_info_test.go index 8d390663fe..54a2ff43c8 100644 --- a/protocol/daemons/server/types/liquidations/daemon_liquidation_info_test.go +++ b/protocol/daemons/server/types/liquidations/daemon_liquidation_info_test.go @@ -25,7 +25,7 @@ func TestLiquidatableSubaccountIds_Multiple_Reads(t *testing.T) { constants.Alice_Num1, constants.Bob_Num0, } - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds, 1) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) @@ -39,7 +39,7 @@ func TestNegativeTncSubaccounts_Multiple_Reads(t *testing.T) { constants.Alice_Num1, constants.Bob_Num0, } - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds, 1) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) @@ -60,7 +60,7 @@ func TestSubaccountsWithOpenPositions_Multiple_Reads(t *testing.T) { } input := []clobtypes.SubaccountOpenPositionInfo{info} - ls.UpdateSubaccountsWithPositions(input) + ls.UpdateSubaccountsWithPositions(input, 1) expected := []satypes.SubaccountId{ constants.Alice_Num1, @@ -78,19 +78,25 @@ func TestLiquidatableSubaccountIds_Multiple_Writes(t *testing.T) { expectedSubaccountIds := []satypes.SubaccountId{ constants.Alice_Num1, } - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.Update(1, expectedSubaccountIds, []satypes.SubaccountId{}, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{ + constants.Alice_Num1, constants.Bob_Num0, } - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.Update( + 1, + []satypes.SubaccountId{constants.Bob_Num0}, + []satypes.SubaccountId{}, + []clobtypes.SubaccountOpenPositionInfo{}, + ) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{ constants.Carl_Num0, } - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.Update(2, expectedSubaccountIds, []satypes.SubaccountId{}, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) } @@ -101,19 +107,25 @@ func TestNegativeTncSubaccounts_Multiple_Writes(t *testing.T) { expectedSubaccountIds := []satypes.SubaccountId{ constants.Alice_Num1, } - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.Update(1, []satypes.SubaccountId{}, expectedSubaccountIds, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{ + constants.Alice_Num1, constants.Bob_Num0, } - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.Update( + 1, + []satypes.SubaccountId{}, + []satypes.SubaccountId{constants.Bob_Num0}, + []clobtypes.SubaccountOpenPositionInfo{}, + ) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{ constants.Carl_Num0, } - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.Update(2, []satypes.SubaccountId{}, expectedSubaccountIds, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) } @@ -132,7 +144,7 @@ func TestSubaccountsWithOpenPositions_Multiple_Writes(t *testing.T) { } input := []clobtypes.SubaccountOpenPositionInfo{info} - ls.UpdateSubaccountsWithPositions(input) + ls.Update(1, []satypes.SubaccountId{}, []satypes.SubaccountId{}, input) expected := []satypes.SubaccountId{ constants.Alice_Num1, constants.Bob_Num0, @@ -150,9 +162,11 @@ func TestSubaccountsWithOpenPositions_Multiple_Writes(t *testing.T) { } input2 := []clobtypes.SubaccountOpenPositionInfo{info2} - ls.UpdateSubaccountsWithPositions(input2) + ls.Update(1, []satypes.SubaccountId{}, []satypes.SubaccountId{}, input2) expected = []satypes.SubaccountId{ + constants.Alice_Num1, constants.Carl_Num0, + constants.Bob_Num0, constants.Dave_Num0, } require.Equal(t, expected, ls.GetSubaccountsWithOpenPositions(0)) @@ -168,7 +182,7 @@ func TestSubaccountsWithOpenPositions_Multiple_Writes(t *testing.T) { } input3 := []clobtypes.SubaccountOpenPositionInfo{info3} - ls.UpdateSubaccountsWithPositions(input3) + ls.Update(2, []satypes.SubaccountId{}, []satypes.SubaccountId{}, input3) expected = []satypes.SubaccountId{ constants.Dave_Num1, constants.Alice_Num1, @@ -183,11 +197,11 @@ func TestLiquidatableSubaccountIds_Empty_Update(t *testing.T) { expectedSubaccountIds := []satypes.SubaccountId{ constants.Alice_Num1, } - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.Update(1, expectedSubaccountIds, []satypes.SubaccountId{}, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetLiquidatableSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{} - ls.UpdateLiquidatableSubaccountIds(expectedSubaccountIds) + ls.Update(2, expectedSubaccountIds, []satypes.SubaccountId{}, []clobtypes.SubaccountOpenPositionInfo{}) require.Empty(t, ls.GetLiquidatableSubaccountIds()) } @@ -198,11 +212,11 @@ func TestNegativeTnc_Empty_Update(t *testing.T) { expectedSubaccountIds := []satypes.SubaccountId{ constants.Alice_Num1, } - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.Update(1, []satypes.SubaccountId{}, expectedSubaccountIds, []clobtypes.SubaccountOpenPositionInfo{}) require.Equal(t, expectedSubaccountIds, ls.GetNegativeTncSubaccountIds()) expectedSubaccountIds = []satypes.SubaccountId{} - ls.UpdateNegativeTncSubaccountIds(expectedSubaccountIds) + ls.Update(2, []satypes.SubaccountId{}, expectedSubaccountIds, []clobtypes.SubaccountOpenPositionInfo{}) require.Empty(t, ls.GetNegativeTncSubaccountIds()) } @@ -220,7 +234,7 @@ func TestSubaccountsWithOpenPosition_Empty_Update(t *testing.T) { }, } input := []clobtypes.SubaccountOpenPositionInfo{info} - ls.UpdateSubaccountsWithPositions(input) + ls.Update(1, []satypes.SubaccountId{}, []satypes.SubaccountId{}, input) expected := []satypes.SubaccountId{ constants.Alice_Num1, constants.Bob_Num0, @@ -228,6 +242,6 @@ func TestSubaccountsWithOpenPosition_Empty_Update(t *testing.T) { require.Equal(t, expected, ls.GetSubaccountsWithOpenPositions(0)) input2 := []clobtypes.SubaccountOpenPositionInfo{} - ls.UpdateSubaccountsWithPositions(input2) + ls.Update(2, []satypes.SubaccountId{}, []satypes.SubaccountId{}, input2) require.Empty(t, ls.GetSubaccountsWithOpenPositions(0)) } diff --git a/protocol/x/clob/abci_test.go b/protocol/x/clob/abci_test.go index 86742726c0..661b8bd4e0 100644 --- a/protocol/x/clob/abci_test.go +++ b/protocol/x/clob/abci_test.go @@ -1265,7 +1265,10 @@ func TestPrepareCheckState(t *testing.T) { } // Set the liquidatable subaccount IDs. - ks.ClobKeeper.DaemonLiquidationInfo.UpdateLiquidatableSubaccountIds(tc.liquidatableSubaccounts) + ks.ClobKeeper.DaemonLiquidationInfo.UpdateLiquidatableSubaccountIds( + tc.liquidatableSubaccounts, + uint32(ctx.BlockHeight()), + ) // Run the test. clob.PrepareCheckState( diff --git a/protocol/x/clob/e2e/withdrawal_gating_test.go b/protocol/x/clob/e2e/withdrawal_gating_test.go index 6a15e7d552..93fbafc74b 100644 --- a/protocol/x/clob/e2e/withdrawal_gating_test.go +++ b/protocol/x/clob/e2e/withdrawal_gating_test.go @@ -412,6 +412,7 @@ func TestWithdrawalGating_NegativeTncSubaccount_BlocksThenUnblocks(t *testing.T) } _, err := tApp.App.Server.LiquidateSubaccounts(ctx, &api.LiquidateSubaccountsRequest{ + BlockHeight: 3, LiquidatableSubaccountIds: tc.liquidatableSubaccountIds, NegativeTncSubaccountIds: tc.negativeTncSubaccountIds, SubaccountOpenPositionInfo: clobtest.GetOpenPositionsFromSubaccounts(tc.subaccounts), @@ -513,6 +514,8 @@ func TestWithdrawalGating_NegativeTncSubaccount_BlocksThenUnblocks(t *testing.T) // unblocked after the withdrawal gating period passes. if tc.expectedErr != "" { _, err = tApp.App.Server.LiquidateSubaccounts(ctx, &api.LiquidateSubaccountsRequest{ + BlockHeight: tc.expectedNegativeTncSubaccountSeenAtBlock[tc.gatedPerpetualId] + + satypes.WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS, LiquidatableSubaccountIds: tc.liquidatableSubaccountIds, NegativeTncSubaccountIds: []satypes.SubaccountId{}, SubaccountOpenPositionInfo: clobtest.GetOpenPositionsFromSubaccounts(tc.subaccounts), diff --git a/protocol/x/clob/keeper/deleveraging_test.go b/protocol/x/clob/keeper/deleveraging_test.go index 8823dfb7b1..6b0084e064 100644 --- a/protocol/x/clob/keeper/deleveraging_test.go +++ b/protocol/x/clob/keeper/deleveraging_test.go @@ -867,7 +867,7 @@ func TestOffsetSubaccountPerpetualPosition(t *testing.T) { } positions := clobtest.GetOpenPositionsFromSubaccounts(tc.subaccounts) - ks.ClobKeeper.DaemonLiquidationInfo.UpdateSubaccountsWithPositions(positions) + ks.ClobKeeper.DaemonLiquidationInfo.UpdateSubaccountsWithPositions(positions, uint32(ks.Ctx.BlockHeight())) fills, deltaQuantumsRemaining := ks.ClobKeeper.OffsetSubaccountPerpetualPosition( ks.Ctx, tc.liquidatedSubaccountId, diff --git a/protocol/x/clob/keeper/liquidations_test.go b/protocol/x/clob/keeper/liquidations_test.go index df4e67d04f..89809421b6 100644 --- a/protocol/x/clob/keeper/liquidations_test.go +++ b/protocol/x/clob/keeper/liquidations_test.go @@ -2133,6 +2133,7 @@ func TestPlacePerpetualLiquidation_Deleveraging(t *testing.T) { ks.ClobKeeper.DaemonLiquidationInfo.UpdateSubaccountsWithPositions( clobtest.GetOpenPositionsFromSubaccounts(tc.subaccounts), + uint32(ctx.BlockHeight()), ) for marketId, oraclePrice := range tc.marketIdToOraclePriceOverride {