Merge branch 'development' into feat/rescue-tokens-levered-positions

ionicprotocol · Oct 24, 2024 · ce49e26 · ce49e26
2 parents 1cc04e1 + 9a75758
commit ce49e26
Show file tree

Hide file tree

Showing 195 changed files with 48,683 additions and 4,101 deletions.
diff --git a/.github/workflows/deploy-bots.yaml b/.github/workflows/deploy-bots.yaml
@@ -326,6 +326,8 @@ jobs:
       INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
       LIQUIDATION_DISCORD_WEBHOOK_URL: ${{ secrets.LIQUIDATION_DISCORD_WEBHOOK_URL }}
       PER_DISCORD_WEBHOOK_URL: ${{ secrets.PER_DISCORD_WEBHOOK_URL }}
+      DISCORD_FAILURE_WEBHOOK_URL: ${{secrets.DISCORD_FAILURE_WEBHOOK_URL }}
+      DISCORD_SUCCESS_WEBHOOK_URL: ${{secrets.DISCORD_SUCCESS_WEBHOOK_URL }}
       ORACLES_DISCORD_WEBHOOK_URL: ${{ secrets.ORACLES_DISCORD_WEBHOOK_URL }}
       SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }}
       SENDGRID_EMAIL_TO: ${{ secrets.SENDGRID_EMAIL_TO}}
@@ -379,6 +381,8 @@ jobs:
       LIQUIDATION_DISCORD_WEBHOOK_URL: ${{ secrets.LIQUIDATION_DISCORD_WEBHOOK_URL }}
       ORACLES_DISCORD_WEBHOOK_URL: ${{ secrets.ORACLES_DISCORD_WEBHOOK_URL }}
       PER_DISCORD_WEBHOOK_URL: ${{ secrets.PER_DISCORD_WEBHOOK_URL }}
+      DISCORD_FAILURE_WEBHOOK_URL: ${{secrets.DISCORD_FAILURE_WEBHOOK_URL }}
+      DISCORD_SUCCESS_WEBHOOK_URL: ${{secrets.DISCORD_SUCCESS_WEBHOOK_URL }}
       PYTH_UPDATER_DISCORD_WEBHOOK_URL: ${{ secrets.PYTH_UPDATER_DISCORD_WEBHOOK_URL }}
       SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
       SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }}

diff --git a/ops/Makefile b/ops/Makefile
@@ -71,6 +71,8 @@ prod-plan:
 	export TF_VAR_liquidation_sendgrid_api_key=${SENDGRID_API_KEY} && \
 	export TF_VAR_liquidation_sendgrid_email_to=${SENDGRID_EMAIL_TO} && \
 	export TF_VAR_per_discord_webhook_url=${PER_DISCORD_WEBHOOK_URL} && \
+	export TF_VAR_discord_failure_webhook_url=${DISCORD_FAILURE_WEBHOOK_URL} && \
+	export TF_VAR_discord_success_webhook_url=${DISCORD_SUCCESS_WEBHOOK_URL} && \
 	export TF_VAR_pyth_updater_discord_webhook_url=${PYTH_UPDATER_DISCORD_WEBHOOK_URL} && \
 	export TF_VAR_infura_api_key=${INFURA_API_KEY} && \
 	export TF_VAR_supabase_key=${SUPABASE_KEY} && \
@@ -92,6 +94,8 @@ prod-deploy:
 	export TF_VAR_oracles_discord_webhook_url=${ORACLES_DISCORD_WEBHOOK_URL} && \
 	export TF_VAR_liquidation_discord_webhook_url=${LIQUIDATION_DISCORD_WEBHOOK_URL} && \
 	export TF_VAR_per_discord_webhook_url=${PER_DISCORD_WEBHOOK_URL} && \
+	export TF_VAR_discord_failure_webhook_url=${DISCORD_FAILURE_WEBHOOK_URL} && \
+	export TF_VAR_discord_success_webhook_url=${DISCORD_SUCCESS_WEBHOOK_URL} && \
 	export TF_VAR_liquidation_sendgrid_api_key=${SENDGRID_API_KEY} && \
 	export TF_VAR_liquidation_sendgrid_email_to=${SENDGRID_EMAIL_TO} && \
 	export TF_VAR_pyth_updater_discord_webhook_url=${PYTH_UPDATER_DISCORD_WEBHOOK_URL} && \

diff --git a/ops/modules/bot/main.tf b/ops/modules/bot/main.tf
@@ -52,7 +52,15 @@ resource "aws_ecs_task_definition" "liquidator_bot_ecs_task" {
         {
           name  = "DISCORD_WEBHOOK_URL"
           value = "${var.liquidation_discord_webhook_url}"
-        } 
+        },
+        {
+          name  = "DISCORD_FAILURE_WEBHOOK_URL"
+          value = "${var.discord_failure_webhook_url}"
+        },
+        {
+          name  = "DISCORD_SUCCESS_WEBHOOK_URL"
+          value = "${var.discord_success_webhook_url}"
+        }  
       ] 
     }
   ])

diff --git a/ops/modules/bot/variables.tf b/ops/modules/bot/variables.tf
@@ -82,4 +82,10 @@ variable "region" {
 variable "liquidation_discord_webhook_url" {
   description = "The Discord webhook URL for liquidation notifications."
   type        = string
+}
+variable "discord_failure_webhook_url" {
+  type = string
+}
+variable "discord_success_webhook_url" {
+  type = string
 }
diff --git a/ops/prod/config.tf b/ops/prod/config.tf
@@ -10,6 +10,8 @@ locals {
     NODE_ENV                   = "production",
     UPTIME_LIQUIDATOR_API      = var.uptime_liquidator_api,
     UPTIME_PYTH_UPDATER_API    = var.uptime_pyth_updater_api,
+    DISCORD_SUCCESS_WEBHOOK_URL = var.discord_success_webhook_url,
+    DISCORD_FAILURE_WEBHOOK_URL = var.discord_failure_webhook_url,
   }
 }
 
@@ -21,6 +23,9 @@ locals {
       SENDGRID_API_KEY    = var.liquidation_sendgrid_api_key,
       SENDGRID_EMAIL_TO   = var.liquidation_sendgrid_email_to,
       UPTIME_LIQUIDATOR_API = var.uptime_liquidator_api,
+      DISCORD_SUCCESS_WEBHOOK_URL = var.discord_success_webhook_url,
+      DISCORD_FAILURE_WEBHOOK_URL = var.discord_failure_webhook_url,
+
     }
   )
   liquidation_base_variables = merge(
@@ -30,6 +35,8 @@ locals {
       SENDGRID_API_KEY    = var.liquidation_sendgrid_api_key,
       SENDGRID_EMAIL_TO   = var.liquidation_sendgrid_email_to,
       UPTIME_LIQUIDATOR_API = var.uptime_liquidator_api,
+      DISCORD_SUCCESS_WEBHOOK_URL = var.discord_success_webhook_url,
+      DISCORD_FAILURE_WEBHOOK_URL = var.discord_failure_webhook_url,
     }
   )
   liquidation_optimism_variables = merge(
@@ -39,6 +46,8 @@ locals {
       SENDGRID_API_KEY    = var.liquidation_sendgrid_api_key,
       SENDGRID_EMAIL_TO   = var.liquidation_sendgrid_email_to,
       UPTIME_LIQUIDATOR_API = var.uptime_liquidator_api,
+      DISCORD_SUCCESS_WEBHOOK_URL = var.discord_success_webhook_url,
+      DISCORD_FAILURE_WEBHOOK_URL = var.discord_failure_webhook_url,
     }
   )
   oracle_price_change_verifier_lambda_variables = merge(

diff --git a/ops/prod/mode.tf b/ops/prod/mode.tf
@@ -34,6 +34,8 @@ module "mode_mainnet_liquidator_ecs" {
   ecs_service_name           = var.liquidator_service_name
   desired_count              = var.desired_count
   liquidation_discord_webhook_url = var.liquidation_discord_webhook_url
+  discord_success_webhook_url = var.discord_success_webhook_url
+  discord_failure_webhook_url = var.discord_failure_webhook_url
   subnet_ids                 = ["subnet-0cd439d262800846e"]
   security_group_ids         = ["sg-0a3996557af867ad0"]
   region                     = var.region

diff --git a/ops/prod/variables.tf b/ops/prod/variables.tf
@@ -43,6 +43,12 @@ variable "pyth_updater_discord_webhook_url" {
 variable "liquidation_discord_webhook_url" {
   type = string
 }
+variable "discord_success_webhook_url" {
+  type = string
+}
+variable "discord_failure_webhook_url" {
+  type = string
+}
 variable "per_discord_webhook_url" {
   type = string
 }

diff --git a/packages/bots/liquidator/README.md b/packages/bots/liquidator/README.md
@@ -29,7 +29,7 @@ Export the relevant environment variables:
 export ETHEREUM_ADMIN_ACCOUNT="0x321..."
 export ETHEREUM_ADMIN_PRIVATE_KEY="0x123..."
 export WEB3_HTTP_PROVIDER_URL="https://bsc-mainnet.gateway.pokt.network/v1/lb/xxxxxxxxxx"
-export TARGET_CHAIN_ID=56
+export TARGET_CHAIN_ID=34443
 ```
 
 And run outside docker:

diff --git a/packages/bots/liquidator/src/config.ts b/packages/bots/liquidator/src/config.ts
@@ -7,11 +7,13 @@ const config = {
   chainId: parseInt(process.env.TARGET_CHAIN_ID ?? "34443", 10),
   rpcUrls: process.env.WEB3_HTTP_PROVIDER_URLS
     ? process.env.WEB3_HTTP_PROVIDER_URLS.split(",")
-    : ["https://mainnet.mode.network/"], // Updated to handle multiple RPC URLs
+    : ["https://mode.drpc.org"],
   adminPrivateKey: process.env.ETHEREUM_ADMIN_PRIVATE_KEY ?? "",
   adminAccount: process.env.ETHEREUM_ADMIN_ACCOUNT ?? "",
   excludedComptrollers: process.env.EXCLUDED_COMPTROLLERS ? process.env.EXCLUDED_COMPTROLLERS.split(",") : [],
   discordWebhookUrl: process.env.DISCORD_WEBHOOK_URL ?? "",
+  DISCORD_FAILURE_WEBHOOK_URL: process.env.DISCORD_FAILURE_WEBHOOK_URL || "",
+  DISCORD_SUCCESS_WEBHOOK_URL: process.env.DISCORD_SUCCESS_WEBHOOK_URL || "",
   PER_discordWebhookUrl: process.env.PER_DISCORD_WEBHOOK_URL ?? "",
   sendgridApiKey: process.env.SENDGRID_API_KEY ?? "",
   sendgridEmailTo: process.env.SENDGRID_EMAIL_TO ?? "[email protected]",

diff --git a/packages/bots/liquidator/src/dummy-run.ts b/packages/bots/liquidator/src/dummy-run.ts
@@ -0,0 +1,98 @@
+import { LiquidatablePool } from "@ionicprotocol/sdk";
+import { createPublicClient, createWalletClient, fallback, Hex, http } from "viem";
+import { privateKeyToAccount } from "viem/accounts";
+import { base } from "viem/chains";
+
+import config from "./config";
+import { Liquidator } from "./services";
+import { setUpSdk } from "./utils";
+
+// Mock the liquidatePositions function
+const liquidatePositions = async (liquidator: Liquidator, options?: { blockNumber?: bigint }) => {
+  const blockNumber = options?.blockNumber;
+  console.log("Block number for liquidation:", blockNumber);
+  const { logger } = liquidator.sdk;
+
+  // Hardcoded liquidation data without unsupported properties
+  const dummyLiquidation: LiquidatablePool = {
+    comptroller: "0x8Fb3D4a94D0aA5D6EDaAC3Ed82B59a27f56d923a",
+    liquidations: [
+      {
+        borrower: "0x88dFF072a2ad3880268Ad15BC647Aa4fb71cb7fB",
+        repayAmount: BigInt(3014530), // Adjust based on expected type
+        cErc20: "0x4341620757Bee7EB4553912FaFC963e59C949147",
+        cTokenCollateral: "0x3120B4907851cc9D780eef9aF88ae4d5360175Fd",
+        minProfitAmount: BigInt(0),
+        flashSwapContract: "0x468cC91dF6F669CaE6cdCE766995Bd7874052FBc",
+        redemptionStrategies: ["0x5cA3fd2c285C4138185Ef1BdA7573D415020F3C8"],
+        strategyData: [
+          "0x000000000000000000000000d988097fb8612cc24eec14542bc03424c656005f000000000000000000000000ac48fcf1049668b285f3dc72483df5ae2162f7e8",
+        ],
+        debtFundingStrategies: [],
+        debtFundingStrategiesData: [],
+      },
+      {
+        borrower: "0xF91bb6A963D870EF53Fdd77af3BB341366725dD8",
+        repayAmount: BigInt(4688498), // Adjust based on expected type
+        cErc20: "0xc53edEafb6D502DAEC5A7015D67936CEa0cD0F52",
+        cTokenCollateral: "0xDb8eE6D1114021A94A045956BBeeCF35d13a30F2",
+        minProfitAmount: BigInt(0),
+        flashSwapContract: "0x468cC91dF6F669CaE6cdCE766995Bd7874052FBc",
+        redemptionStrategies: ["0x5cA3fd2c285C4138185Ef1BdA7573D415020F3C8"],
+        strategyData: [
+          "0x000000000000000000000000d988097fb8612cc24eec14542bc03424c656005f000000000000000000000000ac48fcf1049668b285f3dc72483df5ae2162f7e8",
+        ],
+        debtFundingStrategies: [],
+        debtFundingStrategiesData: [],
+      },
+    ],
+  };
+
+  // Log the dummy liquidation details
+  logger.info(
+    `Liquidating pool: ${dummyLiquidation.comptroller} -- ${dummyLiquidation.liquidations.length} liquidations found`
+  );
+
+  // Simulate liquidating the pool using the hardcoded data
+  await liquidator.liquidate(dummyLiquidation);
+
+  logger.info("Dummy liquidation process completed.");
+};
+
+const run = async () => {
+  try {
+    const account = privateKeyToAccount(config.adminPrivateKey as Hex);
+
+    // Create public and wallet clients
+    const client = createPublicClient({
+      chain: base,
+      transport: fallback(config.rpcUrls.map((url) => http(url))),
+    });
+
+    const walletClient = createWalletClient({
+      account,
+      chain: base,
+      transport: fallback(config.rpcUrls.map((url) => http(url))),
+    });
+
+    // Set up the SDK with the correct chain and clients
+    const sdk = setUpSdk(config.chainId, client as any, walletClient);
+
+    // Initialize the Liquidator
+    const liquidator = new Liquidator(sdk);
+
+    sdk.logger.info(`Starting liquidation bot on chain: ${config.chainId}`);
+    sdk.logger.info(
+      `Config for bot: ${JSON.stringify({ ...sdk.chainLiquidationConfig, ...config, adminPrivateKey: "***********" })}`
+    );
+
+    // Run the dummy liquidation process
+    await liquidatePositions(liquidator);
+
+    sdk.logger.info("Dummy liquidation process completed.");
+  } catch (error) {
+    console.error("Error occurred during liquidation:", error);
+  }
+};
+
+run();
diff --git a/packages/bots/liquidator/src/run.ts b/packages/bots/liquidator/src/run.ts
@@ -1,6 +1,5 @@
 import { createPublicClient, createWalletClient, fallback, Hex, http } from "viem";
 import { privateKeyToAccount } from "viem/accounts";
-import { mode } from "viem/chains";
 
 import config from "./config";
 import liquidatePositions from "./liquidatePositions";
@@ -11,17 +10,15 @@ const run = async () => {
   const account = privateKeyToAccount(config.adminPrivateKey as Hex);
 
   const client = createPublicClient({
-    chain: mode,
     transport: fallback(config.rpcUrls.map((url) => http(url))),
   });
 
   const walletClient = createWalletClient({
     account,
-    chain: mode,
     transport: fallback(config.rpcUrls.map((url) => http(url))),
   });
 
-  const sdk = setUpSdk(config.chainId, client, walletClient);
+  const sdk = setUpSdk(config.chainId, client as any, walletClient);
 
   const liquidator = new Liquidator(sdk);
 

diff --git a/packages/bots/liquidator/src/services/discord.ts b/packages/bots/liquidator/src/services/discord.ts
@@ -52,15 +52,14 @@ export class DiscordService {
       if (Date.now() - lastSentMessages > 1000 * 60 * 15 || lastSentMessages === 0) {
         const embed = this.create()
           .setTitle(`${currentLiquidations.length} liquidation(s) failed for comptroller: ${comptroller}`)
-          .addField("Method", currentLiquidations[0].method, true)
-          .addField("Value", currentLiquidations[0].value.toString(), true)
-          .addField("Args", JSON.stringify(currentLiquidations[0].args), false)
+          //.addField("Method", currentLiquidations[0]?,true)
+          .addField("Value", currentLiquidations[0]?.toString(), true)
+          .addField("Args", JSON.stringify(currentLiquidations[0]), false)
           // Max limit of embed size
           .setDescription(`${msg.slice(0, 2000)}... (truncated, check AWS Logs) @everyone`)
           .setTimestamp()
           .setColor(this.errorColor);
         await this.send(embed);
-        this.lastSentMessages.liquidations = { tx: liquidations.liquidations, timestamp: Date.now() };
       }
     }
   }

diff --git a/packages/bots/liquidator/src/services/discordnew.ts b/packages/bots/liquidator/src/services/discordnew.ts
@@ -0,0 +1,83 @@
+import { EncodedLiquidationTx, ErroredPool, LiquidatablePool } from "@ionicprotocol/sdk";
+import { SupportedChains } from "@ionicprotocol/types";
+import { MessageBuilder, Webhook } from "discord-webhook-node";
+import { TransactionReceipt } from "viem";
+
+import config from "../config";
+export type SimplifiedTransactionReceipt = Pick<
+  TransactionReceipt,
+  "transactionHash" | "contractAddress" | "from" | "to" | "status"
+>;
+
+export class DiscordService {
+  lastSentMessages: {
+    erroredPools: { pools: Array<ErroredPool>; timestamp: number };
+    liquidations: { tx: EncodedLiquidationTx[] | null; timestamp: number };
+  };
+  chainId: SupportedChains;
+  private errorColor = 0xff0000;
+  private warningColor = 0xfcdb03;
+  private infoColor = 0x00ff00;
+  private failureHook = new Webhook(config.DISCORD_FAILURE_WEBHOOK_URL);
+  private successHook = new Webhook(config.DISCORD_SUCCESS_WEBHOOK_URL);
+
+  constructor(chainId: SupportedChains) {
+    this.chainId = chainId;
+    this.lastSentMessages = {
+      erroredPools: { pools: [], timestamp: 0 },
+      liquidations: { tx: null, timestamp: 0 },
+    };
+  }
+
+  private createBaseMessage(): MessageBuilder {
+    return new MessageBuilder().addField("Chain", `Chain ID: ${this.chainId} (${SupportedChains[this.chainId]})`, true);
+  }
+
+  // Helper to handle rate limits
+  private async sendWithRetry(hook: Webhook, message: MessageBuilder, retries: number = 3): Promise<void> {
+    let attempt = 0;
+    const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+    while (attempt < retries) {
+      try {
+        await hook.send(message);
+        return; // exit if successful
+      } catch (error: any) {
+        if (error.response?.statusCode === 429) {
+          // Rate limit error
+          const retryAfter = parseInt(error.response?.headers["retry-after"]) * 1000 || 5000;
+          console.warn(`Rate limited. Retrying after ${retryAfter / 1000}s...`);
+          await delay(retryAfter); // Wait before retrying
+        } else {
+          console.error("Failed to send message to Discord:", error);
+          throw error; // Rethrow error if not rate-limiting
+        }
+      }
+      attempt++;
+    }
+    console.error("Failed to send message after maximum retries.");
+  }
+
+  async sendLiquidationSuccess(_successfulTxs: SimplifiedTransactionReceipt[], msg: string): Promise<void> {
+    const baseMessage = this.createBaseMessage();
+    baseMessage.setColor(this.infoColor);
+    baseMessage.setDescription(`**__Successful Liquidations:__**\n${msg}`);
+    await this.sendWithRetry(this.successHook, baseMessage);
+  }
+
+  async sendLiquidationFailure(pool: LiquidatablePool, errorMessage: string): Promise<void> {
+    const baseMessage = this.createBaseMessage();
+    baseMessage.setColor(this.errorColor);
+    baseMessage.setDescription(
+      `Failed Liquidation:\nBorrower: ${pool.liquidations[0].borrower}\nError: ${errorMessage}`
+    );
+    await this.sendWithRetry(this.failureHook, baseMessage);
+  }
+
+  async sendLiquidationFetchingFailure(_erroredPools: any[], msg: string): Promise<void> {
+    const baseMessage = this.createBaseMessage();
+    baseMessage.setColor(this.warningColor);
+    baseMessage.setDescription(`Failed to fetch liquidations:\n${msg}`);
+    await this.sendWithRetry(this.failureHook, baseMessage);
+  }
+}
diff --git a/packages/bots/liquidator/src/services/email.ts b/packages/bots/liquidator/src/services/email.ts
@@ -52,13 +52,13 @@ export class EmailService {
         const message = this.create();
         message.subject = `${currentLiquidations.length} liquidation(s) failed for comptroller: ${comptroller}`;
         message.text = `
-          Method: ${currentLiquidations[0].method}
-          Value: ${currentLiquidations[0].value}
-          Args: ${JSON.stringify(currentLiquidations[0].args)}
+          Method: ${currentLiquidations[0]}
+          Value: ${currentLiquidations[0]}
+          Args: ${JSON.stringify(currentLiquidations[0])}
           ${msg}
         `;
         await this.send(message);
-        this.lastSentMessages.liquidations = { tx: liquidations.liquidations, timestamp: Date.now() };
+        //this.lastSentMessages.liquidations = { tx: liquidations.liquidations, timestamp: Date.now() };
       }
     }
   }