From 787ae8a10e97417ee9be3ded7ecb0d9e15fceecc Mon Sep 17 00:00:00 2001
From: jayy04 <103467857+jayy04@users.noreply.github.com>
Date: Wed, 21 Aug 2024 09:27:24 -0400
Subject: [PATCH] add telemetry and logs for liquidation daemon (#2122)

(cherry picked from commit 50304592eebad8d99b6770b05d3b3a564579a7a9)

# Conflicts:
#	protocol/daemons/liquidation/client/grpc_helper.go
---
 .../daemons/liquidation/client/grpc_helper.go | 48 +++++++++++++++++++
 .../liquidation/client/sub_task_runner.go     | 24 ++++++++--
 protocol/lib/metrics/constants.go             |  1 +
 3 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/protocol/daemons/liquidation/client/grpc_helper.go b/protocol/daemons/liquidation/client/grpc_helper.go
index aa662f0cc8e..be42d1259f8 100644
--- a/protocol/daemons/liquidation/client/grpc_helper.go
+++ b/protocol/daemons/liquidation/client/grpc_helper.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	errorsmod "cosmossdk.io/errors"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	"github.com/cosmos/cosmos-sdk/types/grpc"
 	"github.com/cosmos/cosmos-sdk/types/query"
@@ -241,11 +242,58 @@ func (c *Client) SendLiquidatableSubaccountIds(
 		subaccountOpenPositionInfo = append(subaccountOpenPositionInfo, *openPositionInfoMap[perpetualId])
 	}
 
+<<<<<<< HEAD
 	request := &api.LiquidateSubaccountsRequest{
 		BlockHeight:                blockHeight,
 		LiquidatableSubaccountIds:  liquidatableSubaccountIds,
 		NegativeTncSubaccountIds:   negativeTncSubaccountIds,
 		SubaccountOpenPositionInfo: subaccountOpenPositionInfo,
+=======
+	// Break this down to multiple requests if the number of subaccounts is too large.
+
+	// Liquidatable subaccount ids.
+	requests := GenerateLiquidateSubaccountsPaginatedRequests(
+		liquidatableSubaccountIds,
+		blockHeight,
+		pageLimit,
+	)
+
+	// Negative TNC subaccount ids.
+	requests = append(
+		requests,
+		GenerateNegativeTNCSubaccountsPaginatedRequests(
+			negativeTncSubaccountIds,
+			blockHeight,
+			pageLimit,
+		)...,
+	)
+
+	// Subaccount open position info.
+	requests = append(
+		requests,
+		GenerateSubaccountOpenPositionPaginatedRequests(
+			subaccountOpenPositionInfo,
+			blockHeight,
+			pageLimit,
+		)...,
+	)
+
+	telemetry.ModuleSetGauge(
+		metrics.LiquidationDaemon,
+		float32(len(requests)),
+		metrics.NumRequests,
+		metrics.Count,
+	)
+
+	for _, req := range requests {
+		if _, err := c.LiquidationServiceClient.LiquidateSubaccounts(ctx, req); err != nil {
+			return errorsmod.Wrapf(
+				err,
+				"failed to send liquidatable subaccount ids to protocol at block height %d",
+				blockHeight,
+			)
+		}
+>>>>>>> 50304592 (add telemetry and logs for liquidation daemon (#2122))
 	}
 
 	if _, err := c.LiquidationServiceClient.LiquidateSubaccounts(ctx, request); err != nil {
diff --git a/protocol/daemons/liquidation/client/sub_task_runner.go b/protocol/daemons/liquidation/client/sub_task_runner.go
index 506d013a9c8..b4bca3514fd 100644
--- a/protocol/daemons/liquidation/client/sub_task_runner.go
+++ b/protocol/daemons/liquidation/client/sub_task_runner.go
@@ -119,13 +119,21 @@ func (c *Client) FetchApplicationStateAtBlockHeight(
 	// Subaccounts
 	subaccounts, err = c.GetAllSubaccounts(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch subaccounts at block height %d",
+			blockHeight,
+		)
 	}
 
 	// Market prices
 	marketPrices, err := c.GetAllMarketPrices(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch market prices at block height %d",
+			blockHeight,
+		)
 	}
 	marketPricesMap := lib.UniqueSliceToMap(marketPrices, func(m pricestypes.MarketPrice) uint32 {
 		return m.Id
@@ -134,13 +142,21 @@ func (c *Client) FetchApplicationStateAtBlockHeight(
 	// Perpetuals
 	perpetuals, err := c.GetAllPerpetuals(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch perpetuals at block height %d",
+			blockHeight,
+		)
 	}
 
 	// Liquidity tiers
 	liquidityTiers, err := c.GetAllLiquidityTiers(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch liquidity tiers at block height %d",
+			blockHeight,
+		)
 	}
 	liquidityTiersMap := lib.UniqueSliceToMap(liquidityTiers, func(l perptypes.LiquidityTier) uint32 {
 		return l.Id
diff --git a/protocol/lib/metrics/constants.go b/protocol/lib/metrics/constants.go
index 8493fa8dc57..0ac766bf5b9 100644
--- a/protocol/lib/metrics/constants.go
+++ b/protocol/lib/metrics/constants.go
@@ -314,6 +314,7 @@ const (
 	GetSubaccountsFromKey                    = "get_subaccounts_from_key"
 	LiquidatableSubaccountIds                = "liquidatable_subaccount_ids"
 	LiquidationDaemon                        = "liquidation_daemon"
+	NumRequests                              = "num_requests"
 	NegativeTncSubaccountIds                 = "negative_tnc_subaccount_ids"
 	PageLimit                                = "page_limit"
 	SendLiquidatableSubaccountIds            = "send_liquidatable_subaccount_ids"