Add Orderbook Mid Price Cache (backport #2338)

indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts

@@ -0,0 +1,136 @@
+import { deleteAllAsync } from '../../src/helpers/redis';
+import { redis as client } from '../helpers/utils';
+import {
+  setPrice,
+  getMedianPrice,
+  ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX,
+} from '../../src/caches/orderbook-mid-prices-cache';
+
+describe('orderbook-mid-prices-cache', () => {
+  const ticker: string = 'BTC-USD';
+
+  beforeEach(async () => {
+    await deleteAllAsync(client);
+  });
+
+  describe('setPrice', () => {
+    it('sets a price for a ticker', async () => {
+      await setPrice(client, ticker, '50000');
+
+      await client.zrange(
+        `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`,
+        0,
+        -1,
+        (_: any, response: string[]) => {
+          expect(response[0]).toBe('50000');
+        },
+      );
+    });
+
+    it('sets multiple prices for a ticker', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '50000'),
+        setPrice(client, ticker, '51000'),
+        setPrice(client, ticker, '49000'),
+      ]);
+
+      await client.zrange(
+        `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`,
+        0,
+        -1,
+        (_: any, response: string[]) => {
+          expect(response).toEqual(['49000', '50000', '51000']);
+        },
+      );
+    });
+  });
+
+  describe('getMedianPrice', () => {
+    it('returns null when no prices are set', async () => {
+      const result = await getMedianPrice(client, ticker);
+      expect(result).toBeNull();
+    });
+
+    it('returns the median price for odd number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '50000'),
+        setPrice(client, ticker, '51000'),
+        setPrice(client, ticker, '49000'),
+      ]);
+
+      const result = await getMedianPrice(client, ticker);
+      expect(result).toBe('50000');
+    });
+
+    it('returns the median price for even number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '50000'),
+        setPrice(client, ticker, '51000'),
+        setPrice(client, ticker, '49000'),
+        setPrice(client, ticker, '52000'),
+      ]);
+
+      const result = await getMedianPrice(client, ticker);
+      expect(result).toBe('50500');
+    });
+
+    it('returns the correct median price after 5 seconds', async () => {
+      jest.useFakeTimers();
+
+      const nowSeconds = Math.floor(Date.now() / 1000);
+      jest.setSystemTime(nowSeconds * 1000);
+
+      await Promise.all([
+        setPrice(client, ticker, '50000'),
+        setPrice(client, ticker, '51000'),
+      ]);
+
+      jest.advanceTimersByTime(6000); // Advance time by 6 seconds
+      await Promise.all([
+        setPrice(client, ticker, '49000'),
+        setPrice(client, ticker, '48000'),
+        setPrice(client, ticker, '52000'),
+        setPrice(client, ticker, '53000'),
+      ]);
+
+      const result = await getMedianPrice(client, ticker);
+      expect(result).toBe('50500');
+
+      jest.useRealTimers();
+    });
+
+    it('returns the correct median price for small numbers with even number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '0.00000000002345'),
+        setPrice(client, ticker, '0.00000000002346'),
+      ]);
+
+      const midPrice1 = await getMedianPrice(client, ticker);
+      expect(midPrice1).toEqual('0.000000000023455');
+    });
+
+    it('returns the correct median price for small numbers with odd number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '0.00000000001'),
+        setPrice(client, ticker, '0.00000000002'),
+        setPrice(client, ticker, '0.00000000003'),
+        setPrice(client, ticker, '0.00000000004'),
+        setPrice(client, ticker, '0.00000000005'),
+      ]);
+
+      const midPrice1 = await getMedianPrice(client, ticker);
+      expect(midPrice1).toEqual('0.00000000003');
+
+      await deleteAllAsync(client);
+
+      await Promise.all([
+        setPrice(client, ticker, '0.00000847007'),
+        setPrice(client, ticker, '0.00000847006'),
+        setPrice(client, ticker, '0.00000847008'),
+      ]);
+
+      const midPrice2 = await getMedianPrice(client, ticker);
+      expect(midPrice2).toEqual('0.00000847007');
+    });
+  });
+});

indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts

@@ -0,0 +1,127 @@
+import Big from 'big.js';
+import { Callback, RedisClient } from 'redis';
+
+import {
+  addMarketPriceScript,
+  getMarketMedianScript,
+} from './scripts';
+
+// Cache of orderbook prices for each clob pair
+// Each price is cached for a 5 second window and in a ZSET
+export const ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX: string = 'v4/orderbook_mid_prices/';
+
+/**
+ * Generates a cache key for a given ticker's orderbook mid price.
+ * @param ticker The ticker symbol
+ * @returns The cache key string
+ */
+function getOrderbookMidPriceCacheKey(ticker: string): string {
+  return `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`;
+}
+
+/**
+ * Adds a price to the market prices cache for a given ticker.
+ * Uses a Lua script to add the price with a timestamp to a sorted set in Redis.
+ * @param client The Redis client
+ * @param ticker The ticker symbol
+ * @param price The price to be added
+ * @returns A promise that resolves when the operation is complete
+ */
+export async function setPrice(
+  client: RedisClient,
+  ticker: string,
+  price: string,
+): Promise<void> {
+  // Number of keys for the lua script.
+  const numKeys: number = 1;
+
+  let evalAsync: (
+    marketCacheKey: string,
+  ) => Promise<void> = (marketCacheKey) => {
+
+    return new Promise<void>((resolve, reject) => {
+      const callback: Callback<void> = (
+        err: Error | null,
+      ) => {
+        if (err) {
+          return reject(err);
+        }
+        return resolve();
+      };
+
+      const nowSeconds = Math.floor(Date.now() / 1000); // Current time in seconds
+      client.evalsha(
+        addMarketPriceScript.hash,
+        numKeys,
+        marketCacheKey,
+        price,
+        nowSeconds,
+        callback,
+      );
+
+    });
+  };
+  evalAsync = evalAsync.bind(client);
+
+  return evalAsync(
+    getOrderbookMidPriceCacheKey(ticker),
+  );
+}
+
+/**
+ * Retrieves the median price for a given ticker from the cache.
+ * Uses a Lua script to fetch either the middle element (for odd number of prices)
+ * or the two middle elements (for even number of prices) from a sorted set in Redis.
+ * If two middle elements are returned, their average is calculated in JavaScript.
+ * @param client The Redis client
+ * @param ticker The ticker symbol
+ * @returns A promise that resolves with the median price as a string, or null if not found
+ */
+export async function getMedianPrice(client: RedisClient, ticker: string): Promise<string | null> {
+  let evalAsync: (
+    marketCacheKey: string,
+  ) => Promise<string[]> = (
+    marketCacheKey,
+  ) => {
+    return new Promise((resolve, reject) => {
+      const callback: Callback<string[]> = (
+        err: Error | null,
+        results: string[],
+      ) => {
+        if (err) {
+          return reject(err);
+        }
+        return resolve(results);
+      };
+
+      client.evalsha(
+        getMarketMedianScript.hash,
+        1,
+        marketCacheKey,
+        callback,
+      );
+    });
+  };
+  evalAsync = evalAsync.bind(client);
+
+  const prices = await evalAsync(
+    getOrderbookMidPriceCacheKey(ticker),
+  );
+
+  if (!prices || prices.length === 0) {
+    return null;
+  }
+
+  if (prices.length === 1) {
+    return Big(prices[0]).toFixed();
+  }
+
+  if (prices.length === 2) {
+    const [price1, price2] = prices.map((price) => {
+      return Big(price);
+    });
+    return price1.plus(price2).div(2).toFixed();
+  }
+
+  return null;
+}

indexer/packages/redis/src/caches/scripts.ts

@@ -63,6 +63,8 @@ export const removeOrderScript: LuaScript = newLuaScript('removeOrder', '../scri
 export const addCanceledOrderIdScript: LuaScript = newLuaScript('addCanceledOrderId', '../scripts/add_canceled_order_id.lua');
 export const addStatefulOrderUpdateScript: LuaScript = newLuaScript('addStatefulOrderUpdate', '../scripts/add_stateful_order_update.lua');
 export const removeStatefulOrderUpdateScript: LuaScript = newLuaScript('removeStatefulOrderUpdate', '../scripts/remove_stateful_order_update.lua');
+export const addMarketPriceScript: LuaScript = newLuaScript('addMarketPrice', '../scripts/add_market_price.lua');
+export const getMarketMedianScript: LuaScript = newLuaScript('getMarketMedianPrice', '../scripts/get_market_median_price.lua');
 
 export const allLuaScripts: LuaScript[] = [
   deleteZeroPriceLevelScript,
@@ -75,4 +77,6 @@ export const allLuaScripts: LuaScript[] = [
   addCanceledOrderIdScript,
   addStatefulOrderUpdateScript,
   removeStatefulOrderUpdateScript,
+  addMarketPriceScript,
+  getMarketMedianScript,
 ];

indexer/packages/redis/src/index.ts

@@ -12,6 +12,7 @@ export * as CanceledOrdersCache from './caches/canceled-orders-cache';
 export * as StatefulOrderUpdatesCache from './caches/stateful-order-updates-cache';
 export * as StateFilledQuantumsCache from './caches/state-filled-quantums-cache';
 export * as LeaderboardPnlProcessedCache from './caches/leaderboard-processed-cache';
+export * as OrderbookMidPricesCache from './caches/orderbook-mid-prices-cache';
 export { placeOrder } from './caches/place-order';
 export { removeOrder } from './caches/remove-order';
 export { updateOrder } from './caches/update-order';

indexer/packages/redis/src/scripts/add_market_price.lua

@@ -0,0 +1,17 @@
+-- Key for the ZSET storing price data
+local priceCacheKey = KEYS[1]
+-- Price to be added
+local price = tonumber(ARGV[1])
+-- Current timestamp
+local nowSeconds = tonumber(ARGV[2])
+-- Time window (5 seconds)
+local fiveSeconds = 5
+
+-- 1. Add the price to the sorted set (score is the current timestamp)
+redis.call("zadd", priceCacheKey, nowSeconds, price)
+
+-- 2. Remove any entries older than 5 seconds
+local cutoffTime = nowSeconds - fiveSeconds
+redis.call("zremrangebyscore", priceCacheKey, "-inf", cutoffTime)
+
+return true

indexer/packages/redis/src/scripts/get_market_median_price.lua

@@ -0,0 +1,22 @@
+-- Key for the sorted set storing price data
+local priceCacheKey = KEYS[1]
+
+-- Get all the prices from the sorted set (ascending order)
+local prices = redis.call('zrange', priceCacheKey, 0, -1)
+
+-- If no prices are found, return nil
+if #prices == 0 then
+  return nil
+end
+
+-- Calculate the middle index
+local middle = math.floor(#prices / 2)
+
+-- Calculate median
+if #prices % 2 == 0 then
+  -- If even, return both prices, division will be handled in Javascript
+  return {prices[middle], prices[middle + 1]}
+else
+  -- If odd, return the middle element
+  return {prices[middle + 1]}
+end