From da16dc7cd5f344514b79843269afd1b0bd643aa8 Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Thu, 19 Sep 2024 16:08:45 -0400 Subject: [PATCH 1/3] Add OrderbookMidPricesCache and Roundtable task --- .../caches/orderbook-mid-prices-cache.test.ts | 106 +++++++++++++++++ .../src/caches/orderbook-mid-prices-cache.ts | 107 ++++++++++++++++++ indexer/packages/redis/src/caches/scripts.ts | 4 + indexer/packages/redis/src/index.ts | 1 + .../redis/src/scripts/add_market_price.lua | 17 +++ .../src/scripts/get_market_median_price.lua | 23 ++++ .../tasks/cache-orderbook-mid-prices.test.ts | 98 ++++++++++++++++ indexer/services/roundtable/src/config.ts | 4 + indexer/services/roundtable/src/index.ts | 9 ++ .../src/tasks/cache-orderbook-mid-prices.ts | 40 +++++++ 10 files changed, 409 insertions(+) create mode 100644 indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts create mode 100644 indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts create mode 100644 indexer/packages/redis/src/scripts/add_market_price.lua create mode 100644 indexer/packages/redis/src/scripts/get_market_median_price.lua create mode 100644 indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts create mode 100644 indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts diff --git a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts new file mode 100644 index 0000000000..70ed134e67 --- /dev/null +++ b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts @@ -0,0 +1,106 @@ +import { deleteAllAsync } from '../../src/helpers/redis'; +import { redis as client } from '../helpers/utils'; +import { + setPrice, + getMedianPrice, + ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX, +} from '../../src/caches/orderbook-mid-prices-cache'; + +describe('orderbook-mid-prices-cache', () => { + const ticker: string = 'BTC-USD'; + + beforeEach(async () => { + await deleteAllAsync(client); + }); + + afterEach(async () => { + await deleteAllAsync(client); + }); + + describe('setPrice', () => { + it('sets a price for a ticker', async () => { + await setPrice(client, ticker, '50000'); + + await client.zrange( + `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, + 0, + -1, + (_: any, response: string[]) => { + expect(response[0]).toBe('50000'); + }, + ); + }); + + it('sets multiple prices for a ticker', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + ]); + + await client.zrange( + `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, + 0, + -1, + (_: any, response: string[]) => { + expect(response).toEqual(['49000', '50000', '51000']); + }, + ); + }); + }); + + describe('getMedianPrice', () => { + it('returns null when no prices are set', async () => { + const result = await getMedianPrice(client, ticker); + expect(result).toBeNull(); + }); + + it('returns the median price for odd number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50000'); + }); + + it('returns the median price for even number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + setPrice(client, ticker, '52000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50500'); + }); + + it('returns the correct median price after 5 seconds', async () => { + jest.useFakeTimers(); + + const nowSeconds = Math.floor(Date.now() / 1000); + jest.setSystemTime(nowSeconds * 1000); + + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + ]); + + jest.advanceTimersByTime(6000); // Advance time by 6 seconds + await Promise.all([ + setPrice(client, ticker, '49000'), + setPrice(client, ticker, '48000'), + setPrice(client, ticker, '52000'), + setPrice(client, ticker, '53000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50500'); + + jest.useRealTimers(); + }); + }); +}); diff --git a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts new file mode 100644 index 0000000000..f2857b70e9 --- /dev/null +++ b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts @@ -0,0 +1,107 @@ +import { Callback, RedisClient } from 'redis'; + +import { + addMarketPriceScript, + getMarketMedianScript, +} from './scripts'; + +// Cache of orderbook prices for each clob pair +// Each price is cached for a 5 second window and in a ZSET +export const ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX: string = 'v4/orderbook_mid_prices/'; + +/** + * Generates a cache key for a given ticker's orderbook mid price. + * @param ticker The ticker symbol + * @returns The cache key string + */ +function getOrderbookMidPriceCacheKey(ticker: string): string { + return `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`; +} + +/** + * Adds a price to the market prices cache for a given ticker. + * Uses a Lua script to add the price with a timestamp to a sorted set in Redis. + * @param client The Redis client + * @param ticker The ticker symbol + * @param price The price to be added + * @returns A promise that resolves when the operation is complete + */ +export async function setPrice( + client: RedisClient, + ticker: string, + price: string, +): Promise { + // Number of keys for the lua script. + const numKeys: number = 1; + + let evalAsync: ( + marketCacheKey: string, + ) => Promise = (marketCacheKey) => { + + return new Promise((resolve, reject) => { + const callback: Callback = ( + err: Error | null, + ) => { + if (err) { + return reject(err); + } + return resolve(); + }; + + const nowSeconds = Math.floor(Date.now() / 1000); // Current time in seconds + client.evalsha( + addMarketPriceScript.hash, + numKeys, + marketCacheKey, + price, + nowSeconds, + callback, + ); + + }); + }; + evalAsync = evalAsync.bind(client); + + return evalAsync( + getOrderbookMidPriceCacheKey(ticker), + ); +} + +/** + * Retrieves the median price for a given ticker from the cache. + * Uses a Lua script to calculate the median price from the sorted set in Redis. + * @param client The Redis client + * @param ticker The ticker symbol + * @returns A promise that resolves with the median price as a string, or null if not found + */ +export async function getMedianPrice(client: RedisClient, ticker: string): Promise { + let evalAsync: ( + marketCacheKey: string, + ) => Promise = ( + marketCacheKey, + ) => { + return new Promise((resolve, reject) => { + const callback: Callback = ( + err: Error | null, + results: string, + ) => { + if (err) { + return reject(err); + } + return resolve(results); + }; + + client.evalsha( + getMarketMedianScript.hash, + 1, + marketCacheKey, + callback, + ); + }); + }; + evalAsync = evalAsync.bind(client); + + return evalAsync( + getOrderbookMidPriceCacheKey(ticker), + ); +} diff --git a/indexer/packages/redis/src/caches/scripts.ts b/indexer/packages/redis/src/caches/scripts.ts index 3e1032c6f2..f4f74bffd5 100644 --- a/indexer/packages/redis/src/caches/scripts.ts +++ b/indexer/packages/redis/src/caches/scripts.ts @@ -63,6 +63,8 @@ export const removeOrderScript: LuaScript = newLuaScript('removeOrder', '../scri export const addCanceledOrderIdScript: LuaScript = newLuaScript('addCanceledOrderId', '../scripts/add_canceled_order_id.lua'); export const addStatefulOrderUpdateScript: LuaScript = newLuaScript('addStatefulOrderUpdate', '../scripts/add_stateful_order_update.lua'); export const removeStatefulOrderUpdateScript: LuaScript = newLuaScript('removeStatefulOrderUpdate', '../scripts/remove_stateful_order_update.lua'); +export const addMarketPriceScript: LuaScript = newLuaScript('addMarketPrice', '../scripts/add_market_price.lua'); +export const getMarketMedianScript: LuaScript = newLuaScript('getMarketMedianPrice', '../scripts/get_market_median_price.lua'); export const allLuaScripts: LuaScript[] = [ deleteZeroPriceLevelScript, @@ -75,4 +77,6 @@ export const allLuaScripts: LuaScript[] = [ addCanceledOrderIdScript, addStatefulOrderUpdateScript, removeStatefulOrderUpdateScript, + addMarketPriceScript, + getMarketMedianScript, ]; diff --git a/indexer/packages/redis/src/index.ts b/indexer/packages/redis/src/index.ts index 2ce64e9b88..5c7aeed103 100644 --- a/indexer/packages/redis/src/index.ts +++ b/indexer/packages/redis/src/index.ts @@ -12,6 +12,7 @@ export * as CanceledOrdersCache from './caches/canceled-orders-cache'; export * as StatefulOrderUpdatesCache from './caches/stateful-order-updates-cache'; export * as StateFilledQuantumsCache from './caches/state-filled-quantums-cache'; export * as LeaderboardPnlProcessedCache from './caches/leaderboard-processed-cache'; +export * as OrderbookMidPricesCache from './caches/orderbook-mid-prices-cache'; export { placeOrder } from './caches/place-order'; export { removeOrder } from './caches/remove-order'; export { updateOrder } from './caches/update-order'; diff --git a/indexer/packages/redis/src/scripts/add_market_price.lua b/indexer/packages/redis/src/scripts/add_market_price.lua new file mode 100644 index 0000000000..0e1467bb31 --- /dev/null +++ b/indexer/packages/redis/src/scripts/add_market_price.lua @@ -0,0 +1,17 @@ +-- Key for the ZSET storing price data +local priceCacheKey = KEYS[1] +-- Price to be added +local price = tonumber(ARGV[1]) +-- Current timestamp +local nowSeconds = tonumber(ARGV[2]) +-- Time window (5 seconds) +local fiveSeconds = 5 + +-- 1. Add the price to the sorted set (score is the current timestamp) +redis.call("zadd", priceCacheKey, nowSeconds, price) + +-- 2. Remove any entries older than 5 seconds +local cutoffTime = nowSeconds - fiveSeconds +redis.call("zremrangebyscore", priceCacheKey, "-inf", cutoffTime) + +return true \ No newline at end of file diff --git a/indexer/packages/redis/src/scripts/get_market_median_price.lua b/indexer/packages/redis/src/scripts/get_market_median_price.lua new file mode 100644 index 0000000000..281da9bed8 --- /dev/null +++ b/indexer/packages/redis/src/scripts/get_market_median_price.lua @@ -0,0 +1,23 @@ +-- Key for the sorted set storing price data +local priceCacheKey = KEYS[1] + +-- Get all the prices from the sorted set (ascending order) +local prices = redis.call('zrange', priceCacheKey, 0, -1) + +-- If no prices are found, return nil +if #prices == 0 then + return nil +end + +-- Calculate the middle index +local middle = math.floor(#prices / 2) + +-- Calculate median +if #prices % 2 == 0 then + -- If even, return the average of the two middle elements + local median = (tonumber(prices[middle]) + tonumber(prices[middle + 1])) / 2 + return tostring(median) +else + -- If odd, return the middle element + return prices[middle + 1] +end diff --git a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts new file mode 100644 index 0000000000..cd0eee3970 --- /dev/null +++ b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts @@ -0,0 +1,98 @@ +import { + dbHelpers, + testConstants, + testMocks, +} from '@dydxprotocol-indexer/postgres'; +import { + OrderbookMidPricesCache, + OrderbookLevelsCache, + redis, +} from '@dydxprotocol-indexer/redis'; +import { redisClient } from '../../src/helpers/redis'; +import runTask from '../../src/tasks/cache-orderbook-mid-prices'; + +jest.mock('@dydxprotocol-indexer/base', () => ({ + ...jest.requireActual('@dydxprotocol-indexer/base'), + logger: { + info: jest.fn(), + error: jest.fn(), + }, +})); + +jest.mock('@dydxprotocol-indexer/redis', () => ({ + ...jest.requireActual('@dydxprotocol-indexer/redis'), + OrderbookLevelsCache: { + getOrderBookMidPrice: jest.fn(), + }, +})); + +describe('cache-orderbook-mid-prices', () => { + beforeAll(async () => { + await dbHelpers.migrate(); + }); + + beforeEach(async () => { + await dbHelpers.clearData(); + await redis.deleteAllAsync(redisClient); + await testMocks.seedData(); + }); + + afterAll(async () => { + await dbHelpers.teardown(); + jest.resetAllMocks(); + }); + + it('caches mid prices for all markets', async () => { + const market1 = testConstants.defaultPerpetualMarket; + const market2 = testConstants.defaultPerpetualMarket2; + + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockResolvedValueOnce('100.5'); // For market1 + mockGetOrderBookMidPrice.mockResolvedValueOnce('200.75'); // For market2 + + await runTask(); + + // Check if the mock was called with the correct arguments + expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market1.ticker, redisClient); + expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market2.ticker, redisClient); + + // Check if prices were cached correctly + const price1 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market1.ticker); + const price2 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market2.ticker); + + expect(price1).toBe('100.5'); + expect(price2).toBe('200.75'); + }); + + it('handles undefined prices', async () => { + const market = testConstants.defaultPerpetualMarket; + + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockResolvedValueOnce(undefined); + + await runTask(); + + const price = await OrderbookMidPricesCache.getMedianPrice(redisClient, market.ticker); + expect(price).toBeNull(); + + // Check that a log message was created + expect(jest.requireMock('@dydxprotocol-indexer/base').logger.info).toHaveBeenCalledWith({ + at: 'cache-orderbook-mid-prices#runTask', + message: `undefined price for ${market.ticker}`, + }); + }); + + it('handles errors', async () => { + // Mock OrderbookLevelsCache.getOrderBookMidPrice to throw an error + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockRejectedValueOnce(new Error('Test error')); + + await runTask(); + + expect(jest.requireMock('@dydxprotocol-indexer/base').logger.error).toHaveBeenCalledWith({ + at: 'cache-orderbook-mid-prices#runTask', + message: 'Test error', + error: expect.any(Error), + }); + }); +}); diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index a8ec2cb87b..9f0f00c487 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -60,6 +60,7 @@ export const configSchema = { LOOPS_ENABLED_UPDATE_WALLET_TOTAL_VOLUME: parseBoolean({ default: true }), LOOPS_ENABLED_UPDATE_AFFILIATE_INFO: parseBoolean({ default: true }), LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS: parseBoolean({ default: true }), + LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES: parseBoolean({ default: true }), // Loop Timing LOOPS_INTERVAL_MS_MARKET_UPDATER: parseInteger({ @@ -137,6 +138,9 @@ export const configSchema = { LOOPS_INTERVAL_MS_DELETE_FIREBASE_NOTIFICATION_TOKENS_MONTHLY: parseInteger({ default: 30 * ONE_DAY_IN_MILLISECONDS, }), + LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES: parseInteger({ + default: ONE_SECOND_IN_MILLISECONDS, + }), // Start delay START_DELAY_ENABLED: parseBoolean({ default: true }), diff --git a/indexer/services/roundtable/src/index.ts b/indexer/services/roundtable/src/index.ts index f52903ac19..bfdee334c7 100644 --- a/indexer/services/roundtable/src/index.ts +++ b/indexer/services/roundtable/src/index.ts @@ -10,6 +10,7 @@ import { connect as connectToRedis, } from './helpers/redis'; import aggregateTradingRewardsTasks from './tasks/aggregate-trading-rewards'; +import cacheOrderbookMidPrices from './tasks/cache-orderbook-mid-prices'; import cancelStaleOrdersTask from './tasks/cancel-stale-orders'; import createLeaderboardTask from './tasks/create-leaderboard'; import createPnlTicksTask from './tasks/create-pnl-ticks'; @@ -272,6 +273,14 @@ async function start(): Promise { ); } + if (config.LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES) { + startLoop( + cacheOrderbookMidPrices, + 'cache_orderbook_mid_prices', + config.LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES, + ); + } + logger.info({ at: 'index', message: 'Successfully started', diff --git a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts new file mode 100644 index 0000000000..644f50df6f --- /dev/null +++ b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts @@ -0,0 +1,40 @@ +import { + logger, +} from '@dydxprotocol-indexer/base'; +import { + PerpetualMarketFromDatabase, + PerpetualMarketTable, +} from '@dydxprotocol-indexer/postgres'; +import { + OrderbookMidPricesCache, + OrderbookLevelsCache, +} from '@dydxprotocol-indexer/redis'; + +import { redisClient } from '../helpers/redis'; + +/** + * Updates OrderbookMidPricesCache with current orderbook mid price for each market + */ +export default async function runTask(): Promise { + const markets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll({}, []); + + for (const market of markets) { + try { + const price = await OrderbookLevelsCache.getOrderBookMidPrice(market.ticker, redisClient); + if (price) { + await OrderbookMidPricesCache.setPrice(redisClient, market.ticker, price); + } else { + logger.info({ + at: 'cache-orderbook-mid-prices#runTask', + message: `undefined price for ${market.ticker}`, + }); + } + } catch (error) { + logger.error({ + at: 'cache-orderbook-mid-prices#runTask', + message: error.message, + error, + }); + } + } +} From 843aea33430146a96ca80d78cbdcdd770910f7df Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Thu, 19 Sep 2024 16:09:16 -0400 Subject: [PATCH 2/3] Use OrderbookMidPrice cache in candles generator --- .../__tests__/lib/candles-generator.test.ts | 53 +++++++++---------- .../ender/src/lib/candles-generator.ts | 6 +-- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index d1c257b80b..d58ebd0d2a 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -18,7 +18,6 @@ import { testMocks, Transaction, helpers, - OrderSide, } from '@dydxprotocol-indexer/postgres'; import { CandleMessage, CandleMessage_Resolution } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -32,9 +31,11 @@ import { KafkaPublisher } from '../../src/lib/kafka-publisher'; import { ConsolidatedKafkaEvent } from '../../src/lib/types'; import { defaultTradeContent, defaultTradeKafkaEvent } from '../helpers/constants'; import { contentToSingleTradeMessage, createConsolidatedKafkaEventFromTrade } from '../helpers/kafka-publisher-helpers'; -import { updatePriceLevel } from '../helpers/redis-helpers'; import { redisClient } from '../../src/helpers/redis/redis-controller'; -import { redis } from '@dydxprotocol-indexer/redis'; +import { + redis, + OrderbookMidPricesCache, +} from '@dydxprotocol-indexer/redis'; describe('candleHelper', () => { beforeAll(async () => { @@ -113,9 +114,12 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - // Create Orderbook levels to set orderbookMidPrice open & close - await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); + const ticker = 'BTC-USD'; + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, ticker, '100000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '105000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '110000'), + ]); await runUpdateCandles(publisher); @@ -155,8 +159,12 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - await updatePriceLevel('BTC-USD', '80000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '81000', OrderSide.SELL); + const ticker = 'BTC-USD'; + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, ticker, '80000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '81000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '80500'), + ]); // Create Perpetual Position to set open position const openInterest: string = '100'; @@ -427,9 +435,7 @@ describe('candleHelper', () => { containsKafkaMessages: boolean = true, orderbookMidPrice: number, ) => { - const midPriceSpread = 10; - await updatePriceLevel('BTC-USD', String(orderbookMidPrice + midPriceSpread), OrderSide.SELL); - await updatePriceLevel('BTC-USD', String(orderbookMidPrice - midPriceSpread), OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', orderbookMidPrice.toFixed()); if (initialCandle !== undefined) { await CandleTable.create(initialCandle); @@ -494,9 +500,7 @@ describe('candleHelper', () => { ); await startCandleCache(); - // Update Orderbook levels - await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); - await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([ @@ -594,9 +598,7 @@ describe('candleHelper', () => { ); await startCandleCache(); - // Update Orderbook levels - await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); - await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([]); @@ -660,22 +662,19 @@ describe('candleHelper', () => { }); it('successfully creates an orderbook price map for each market', async () => { - await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); - - await updatePriceLevel('ISO-USD', '110000', OrderSide.BUY); - await updatePriceLevel('ISO-USD', '120000', OrderSide.SELL); - - await updatePriceLevel('ETH-USD', '100000', OrderSide.BUY); - await updatePriceLevel('ETH-USD', '200000', OrderSide.SELL); + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '105000'), + OrderbookMidPricesCache.setPrice(redisClient, 'ISO-USD', '115000'), + OrderbookMidPricesCache.setPrice(redisClient, 'ETH-USD', '150000'), + ]); const map = await getOrderbookMidPriceMap(); expect(map).toEqual({ 'BTC-USD': '105000', 'ETH-USD': '150000', 'ISO-USD': '115000', - 'ISO2-USD': undefined, - 'SHIB-USD': undefined, + 'ISO2-USD': null, + 'SHIB-USD': null, }); }); }); diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index d7dd7bba34..b232a66eb0 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -20,7 +20,7 @@ import { TradeMessageContents, helpers, } from '@dydxprotocol-indexer/postgres'; -import { OrderbookLevelsCache } from '@dydxprotocol-indexer/redis'; +import { OrderbookMidPricesCache } from '@dydxprotocol-indexer/redis'; import { CandleMessage } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; import _ from 'lodash'; @@ -538,9 +538,9 @@ export async function getOrderbookMidPriceMap(): Promise<{ [ticker: string]: Ord const perpetualMarkets = Object.values(perpetualMarketRefresher.getPerpetualMarketsMap()); const promises = perpetualMarkets.map(async (perpetualMarket: PerpetualMarketFromDatabase) => { - const price = await OrderbookLevelsCache.getOrderBookMidPrice( - perpetualMarket.ticker, + const price = await OrderbookMidPricesCache.getMedianPrice( redisClient, + perpetualMarket.ticker, ); return { [perpetualMarket.ticker]: price === undefined ? undefined : price }; }); From 5a6ddd4a35dead6608518d81f35446590663a5ac Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Tue, 24 Sep 2024 13:22:59 -0400 Subject: [PATCH 3/3] Return values from redis without division, perform division in javascript --- .../caches/orderbook-mid-prices-cache.test.ts | 38 +++++++++++++++++-- .../src/caches/orderbook-mid-prices-cache.ts | 30 ++++++++++++--- .../src/scripts/get_market_median_price.lua | 7 ++-- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts index 70ed134e67..5dfd662f68 100644 --- a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts +++ b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts @@ -13,10 +13,6 @@ describe('orderbook-mid-prices-cache', () => { await deleteAllAsync(client); }); - afterEach(async () => { - await deleteAllAsync(client); - }); - describe('setPrice', () => { it('sets a price for a ticker', async () => { await setPrice(client, ticker, '50000'); @@ -102,5 +98,39 @@ describe('orderbook-mid-prices-cache', () => { jest.useRealTimers(); }); + + it('returns the correct median price for small numbers with even number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '0.00000000002345'), + setPrice(client, ticker, '0.00000000002346'), + ]); + + const midPrice1 = await getMedianPrice(client, ticker); + expect(midPrice1).toEqual('0.000000000023455'); + }); + + it('returns the correct median price for small numbers with odd number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '0.00000000001'), + setPrice(client, ticker, '0.00000000002'), + setPrice(client, ticker, '0.00000000003'), + setPrice(client, ticker, '0.00000000004'), + setPrice(client, ticker, '0.00000000005'), + ]); + + const midPrice1 = await getMedianPrice(client, ticker); + expect(midPrice1).toEqual('0.00000000003'); + + await deleteAllAsync(client); + + await Promise.all([ + setPrice(client, ticker, '0.00000847007'), + setPrice(client, ticker, '0.00000847006'), + setPrice(client, ticker, '0.00000847008'), + ]); + + const midPrice2 = await getMedianPrice(client, ticker); + expect(midPrice2).toEqual('0.00000847007'); + }); }); }); diff --git a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts index f2857b70e9..ece95a3ca2 100644 --- a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts +++ b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts @@ -1,3 +1,4 @@ +import Big from 'big.js'; import { Callback, RedisClient } from 'redis'; import { @@ -69,7 +70,9 @@ export async function setPrice( /** * Retrieves the median price for a given ticker from the cache. - * Uses a Lua script to calculate the median price from the sorted set in Redis. + * Uses a Lua script to fetch either the middle element (for odd number of prices) + * or the two middle elements (for even number of prices) from a sorted set in Redis. + * If two middle elements are returned, their average is calculated in JavaScript. * @param client The Redis client * @param ticker The ticker symbol * @returns A promise that resolves with the median price as a string, or null if not found @@ -77,13 +80,13 @@ export async function setPrice( export async function getMedianPrice(client: RedisClient, ticker: string): Promise { let evalAsync: ( marketCacheKey: string, - ) => Promise = ( + ) => Promise = ( marketCacheKey, ) => { return new Promise((resolve, reject) => { - const callback: Callback = ( + const callback: Callback = ( err: Error | null, - results: string, + results: string[], ) => { if (err) { return reject(err); @@ -101,7 +104,24 @@ export async function getMedianPrice(client: RedisClient, ticker: string): Promi }; evalAsync = evalAsync.bind(client); - return evalAsync( + const prices = await evalAsync( getOrderbookMidPriceCacheKey(ticker), ); + + if (!prices || prices.length === 0) { + return null; + } + + if (prices.length === 1) { + return Big(prices[0]).toFixed(); + } + + if (prices.length === 2) { + const [price1, price2] = prices.map((price) => { + return Big(price); + }); + return price1.plus(price2).div(2).toFixed(); + } + + return null; } diff --git a/indexer/packages/redis/src/scripts/get_market_median_price.lua b/indexer/packages/redis/src/scripts/get_market_median_price.lua index 281da9bed8..a318296f20 100644 --- a/indexer/packages/redis/src/scripts/get_market_median_price.lua +++ b/indexer/packages/redis/src/scripts/get_market_median_price.lua @@ -14,10 +14,9 @@ local middle = math.floor(#prices / 2) -- Calculate median if #prices % 2 == 0 then - -- If even, return the average of the two middle elements - local median = (tonumber(prices[middle]) + tonumber(prices[middle + 1])) / 2 - return tostring(median) + -- If even, return both prices, division will be handled in Javascript + return {prices[middle], prices[middle + 1]} else -- If odd, return the middle element - return prices[middle + 1] + return {prices[middle + 1]} end