chore: add more logs for OOM search

src/domain/events/index.ts

@@ -1,18 +1,18 @@
 import {
-  toConditionalOrderParams,
+  getAreConditionalOrderParamsEqual,
   getLogger,
   handleExecutionError,
   metrics,
+  toConditionalOrderParams,
 } from "../../utils";
 import { BytesLike, ethers } from "ethers";
 
 import {
   ComposableCoW,
-  ComposableCoWInterface,
+  ComposableCoW__factory,
   ConditionalOrderCreatedEvent,
   IConditionalOrder,
   MerkleRootSetEvent,
-  ComposableCoW__factory,
   Owner,
   Proof,
   Registry,
@@ -21,6 +21,8 @@ import { ConditionalOrder, ConditionalOrderParams } from "@cowprotocol/cow-sdk";
 
 import { ChainContext } from "../../services/chain";
 
+const composableCow = ComposableCoW__factory.createInterface();
+
 /**
  * Listens to these events on the `ComposableCoW` contract:
  * - `ConditionalOrderCreated`
@@ -48,19 +50,15 @@ async function _addContract(
   event: ConditionalOrderCreatedEvent
 ) {
   const log = getLogger("addContract:_addContract");
-  const composableCow = ComposableCoW__factory.createInterface();
   const { registry } = context;
   const { transactionHash: tx, blockNumber } = event;
 
   // Process the logs
   let hasErrors = false;
   let numContractsAdded = 0;
 
-  const { error, added } = await _registerNewOrder(
-    event,
-    composableCow,
-    registry
-  );
+  const { error, added } = await registerNewOrder(event, registry);
+
   if (added) {
     metrics.ownersTotal.labels(context.chainId.toString()).inc();
     numContractsAdded++;
@@ -69,6 +67,7 @@ async function _addContract(
       `Failed to register Smart Order from tx ${tx} on block ${blockNumber}. Error: ${error}`
     );
   }
+
   hasErrors ||= error;
 
   if (numContractsAdded > 0) {
@@ -86,15 +85,15 @@ async function _addContract(
   }
 }
 
-export async function _registerNewOrder(
+async function registerNewOrder(
   event: ConditionalOrderCreatedEvent | MerkleRootSetEvent,
-  composableCow: ComposableCoWInterface,
   registry: Registry
 ): Promise<{ error: boolean; added: boolean }> {
-  const log = getLogger("addContract:_registerNewOrder");
+  const log = getLogger("addContract:registerNewOrder");
   const { transactionHash: tx } = event;
   const { network } = registry;
   let added = false;
+
   try {
     // Check if the log is a ConditionalOrderCreated event
     if (
@@ -177,13 +176,14 @@ export async function _registerNewOrder(
 /**
  * Attempt to add an owner's conditional order to the registry
  *
+ * @param tx transaction that created the conditional order
  * @param owner to add the conditional order to
  * @param params for the conditional order
  * @param proof for the conditional order (if it is part of a merkle root)
  * @param composableCow address of the contract that emitted the event
  * @param registry of all conditional orders
  */
-export function add(
+function add(
   tx: string,
   owner: Owner,
   params: ConditionalOrderParams,
@@ -206,6 +206,24 @@ export function add(
     // Iterate over the conditionalOrders to make sure that the params are not already in the registry
     for (const conditionalOrder of conditionalOrders?.values() ?? []) {
       // Check if the params are in the conditionalOrder
+      if (conditionalOrder) {
+        const areConditionalOrderParamsEqual =
+          getAreConditionalOrderParamsEqual(conditionalOrder.params, params);
+
+        // TODO: delete this log after testing
+        if (
+          areConditionalOrderParamsEqual &&
+          conditionalOrder.params !== params
+        ) {
+          log.error(
+            "Conditional order params are equal but not the same",
+            conditionalOrder.id,
+            JSON.stringify(params)
+          );
+        }
+      }
+
+      // TODO: this is a shallow comparison, should we do a deep comparison?
       if (conditionalOrder.params === params) {
         exists = true;
         break;
@@ -245,6 +263,7 @@ export function add(
         },
       ])
     );
+
     metrics.activeOwnersTotal.labels(network).inc();
     metrics.activeOrdersTotal.labels(network).inc();
   }
@@ -256,19 +275,20 @@ export function add(
  * @param root the merkle root to check against
  * @param registry of all conditional orders
  */
-export function flush(owner: Owner, root: BytesLike, registry: Registry) {
-  if (registry.ownerOrders.has(owner)) {
-    const conditionalOrders = registry.ownerOrders.get(owner);
-    if (conditionalOrders !== undefined) {
-      for (const conditionalOrder of conditionalOrders.values()) {
-        if (
-          conditionalOrder.proof !== null &&
-          conditionalOrder.proof.merkleRoot !== root
-        ) {
-          // Delete the conditional order
-          conditionalOrders.delete(conditionalOrder);
-        }
-      }
+function flush(owner: Owner, root: BytesLike, registry: Registry) {
+  if (!registry.ownerOrders.has(owner)) return;
+
+  const conditionalOrders = registry.ownerOrders.get(owner);
+
+  if (conditionalOrders === undefined) return;
+
+  for (const conditionalOrder of conditionalOrders.values()) {
+    if (
+      conditionalOrder.proof !== null &&
+      conditionalOrder.proof.merkleRoot !== root
+    ) {
+      // Delete the conditional order
+      conditionalOrders.delete(conditionalOrder);
     }
   }
 }

src/services/chain.ts

@@ -16,7 +16,6 @@ import {
   Multicall3__factory,
   Registry,
   RegistryBlock,
-  blockToRegistryBlock,
 } from "../types";
 import {
   LoggerWithMethods,
@@ -32,7 +31,7 @@ const WATCHDOG_TIMEOUT_DEFAULT_SECS = 30;
 const MULTICALL3 = "0xcA11bde05977b3631167028862bE2a173976CA11";
 const PAGE_SIZE_DEFAULT = 5000;
 
-export const SDK_BACKOFF_NUM_OF_ATTEMPTS = 5;
+const SDK_BACKOFF_NUM_OF_ATTEMPTS = 5;
 
 enum ChainSync {
   /** The chain is currently in the warm-up phase, synchronising from contract genesis or lastBlockProcessed */
@@ -459,6 +458,7 @@ async function processBlock(
   let hasErrors = false;
   for (const event of events) {
     const receipt = await provider.getTransactionReceipt(event.transactionHash);
+
     if (receipt) {
       // run action
       log.debug(`Running "addContract" action for TX ${event.transactionHash}`);
@@ -614,3 +614,11 @@ function getProvider(rpcUrl: string): providers.Provider {
 async function asyncSleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+function blockToRegistryBlock(block: ethers.providers.Block): RegistryBlock {
+  return {
+    number: block.number,
+    timestamp: block.timestamp,
+    hash: block.hash,
+  };
+}

src/types/model.ts

@@ -1,10 +1,10 @@
 import Slack = require("node-slack");
 
-import { BytesLike, ethers } from "ethers";
+import { BytesLike } from "ethers";
 
 import {
-  ConditionalOrderParams,
   ConditionalOrder as ConditionalOrderSdk,
+  ConditionalOrderParams,
   PollResult,
 } from "@cowprotocol/cow-sdk";
 import { DBService } from "../services";
@@ -41,6 +41,7 @@ export type Proof = {
 
 export type OrderUid = BytesLike;
 export type Owner = string;
+
 export enum OrderStatus {
   SUBMITTED = 1,
   FILLED = 2,
@@ -95,27 +96,30 @@ export interface RegistryBlock {
   hash: string;
 }
 
+type OrdersPerOwner = Map<Owner, Set<ConditionalOrder>>;
+
 /**
  * Models the state between executions.
  * Contains a map of owners to conditional orders and the last time we sent an error.
  */
 export class Registry {
   version = CONDITIONAL_ORDER_REGISTRY_VERSION;
-  ownerOrders: Map<Owner, Set<ConditionalOrder>>;
+  ownerOrders: OrdersPerOwner;
   storage: DBService;
   network: string;
   lastNotifiedError: Date | null;
   lastProcessedBlock: RegistryBlock | null;
   readonly logger = getLogger("Registry");
-
   /**
    * Instantiates a registry.
    * @param ownerOrders What map to populate the registry with
    * @param storage storage service to use
    * @param network Which network the registry is for
+   * @param lastNotifiedError The last time we sent an error
+   * @param lastProcessedBlock The last block we processed
    */
   constructor(
-    ownerOrders: Map<Owner, Set<ConditionalOrder>>,
+    ownerOrders: OrdersPerOwner,
     storage: DBService,
     network: string,
     lastNotifiedError: Date | null,
@@ -144,8 +148,9 @@ export class Registry {
 
   /**
    * Load the registry from storage.
-   * @param context from which to load the registry
+   * @param storage to load the registry from
    * @param network that the registry is for
+   * @param genesisBlockNumber the block number of the genesis block
    * @returns a registry instance
    */
   public static async load(
@@ -157,6 +162,7 @@ export class Registry {
     const ownerOrders = await loadOwnerOrders(storage, network).catch(() =>
       createNewOrderMap()
     );
+
     const lastNotifiedError = await db
       .get(getNetworkStorageKey(LAST_NOTIFIED_ERROR_STORAGE_KEY, network))
       .then((isoDate: string | number | Date) =>
@@ -190,11 +196,7 @@ export class Registry {
   }
 
   get numOrders(): number {
-    let count = 0;
-    for (const orders of this.ownerOrders.values()) {
-      count += orders.size; // Count each set's size directly
-    }
-    return count;
+    return getOrdersCountFromOrdersPerOwner(this.ownerOrders);
   }
 
   /**
@@ -259,7 +261,7 @@ export class Registry {
   }
 }
 
-export function _reviver(_key: any, value: any) {
+function reviver(_key: any, value: any) {
   if (typeof value === "object" && value !== null) {
     if (value.dataType === "Map") {
       return new Map(value.value);
@@ -270,7 +272,7 @@ export function _reviver(_key: any, value: any) {
   return value;
 }
 
-export function replacer(_key: any, value: any) {
+function replacer(_key: any, value: any) {
   if (value instanceof Map) {
     return {
       dataType: "Map",
@@ -286,10 +288,12 @@ export function replacer(_key: any, value: any) {
   }
 }
 
+const loadOwnerOrdersLogger = getLogger("loadOwnerOrders");
+
 async function loadOwnerOrders(
   storage: DBService,
   network: string
-): Promise<Map<string, Set<ConditionalOrder>>> {
+): Promise<OrdersPerOwner> {
   // Get the owner orders
   const db = storage.getDB();
   const str = await db.get(
@@ -304,20 +308,33 @@ async function loadOwnerOrders(
   );
 
   // Parse conditional orders registry (for the persisted version, converting it to the last version)
-  return parseConditionalOrders(!!str ? str : undefined, version);
+  const ordersPerOwner = parseConditionalOrders(
+    !!str ? str : undefined,
+    version
+  );
+
+  loadOwnerOrdersLogger.info(
+    `Data size: ${str?.length}`,
+    `Owners: ${ordersPerOwner.size}`,
+    `Total orders: ${getOrdersCountFromOrdersPerOwner(ordersPerOwner)}`,
+    `Network: ${network}`
+  );
+
+  return ordersPerOwner;
 }
 
 function parseConditionalOrders(
   serializedConditionalOrders: string | undefined,
   _version: number | undefined
-): Map<Owner, Set<ConditionalOrder>> {
+): OrdersPerOwner {
   if (!serializedConditionalOrders) {
     return createNewOrderMap();
   }
+
   const ownerOrders: Map<
     Owner,
     Set<Omit<ConditionalOrder, "id"> & { id?: string }>
-  > = JSON.parse(serializedConditionalOrders, _reviver);
+  > = JSON.parse(serializedConditionalOrders, reviver);
 
   // The conditional order id was added in version 2
   if (_version && _version < 2) {
@@ -331,19 +348,21 @@ function parseConditionalOrders(
     }
   }
 
-  return ownerOrders as Map<Owner, Set<ConditionalOrder>>;
+  return ownerOrders as OrdersPerOwner;
 }
 
-function createNewOrderMap(): Map<Owner, Set<ConditionalOrder>> {
+function createNewOrderMap(): OrdersPerOwner {
   return new Map<Owner, Set<ConditionalOrder>>();
 }
 
-export function blockToRegistryBlock(
-  block: ethers.providers.Block
-): RegistryBlock {
-  return {
-    number: block.number,
-    timestamp: block.timestamp,
-    hash: block.hash,
-  };
+function getOrdersCountFromOrdersPerOwner(
+  ordersPerOwner: OrdersPerOwner
+): number {
+  let ordersCount = 0;
+
+  for (const orders of ordersPerOwner.values()) {
+    ordersCount += orders.size;
+  }
+
+  return ordersCount;
 }