From f9c279d183442208a05c47bed45bf555d31988eb Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 28 Aug 2024 15:37:33 -0400 Subject: [PATCH 1/3] create persistent_cache table --- .../postgres/__tests__/helpers/constants.ts | 13 ++++ .../stores/persistent-cache-table.test.ts | 72 +++++++++++++++++++ ...828130730_create_persistent_cache_table.ts | 16 +++++ .../postgres/src/helpers/db-helpers.ts | 1 + indexer/packages/postgres/src/index.ts | 2 + .../src/models/persistent-cache-model.ts | 31 ++++++++ .../postgres/src/types/db-model-types.ts | 5 ++ indexer/packages/postgres/src/types/index.ts | 1 + .../src/types/persistent-cache-types.ts | 14 ++++ .../postgres/src/types/query-types.ts | 5 ++ 10 files changed, 160 insertions(+) create mode 100644 indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts create mode 100644 indexer/packages/postgres/src/models/persistent-cache-model.ts create mode 100644 indexer/packages/postgres/src/types/persistent-cache-types.ts diff --git a/indexer/packages/postgres/__tests__/helpers/constants.ts b/indexer/packages/postgres/__tests__/helpers/constants.ts index 7a8a9a224b..a3cdb30d17 100644 --- a/indexer/packages/postgres/__tests__/helpers/constants.ts +++ b/indexer/packages/postgres/__tests__/helpers/constants.ts @@ -57,6 +57,7 @@ import { TransactionCreateObject, TransferCreateObject, WalletCreateObject, + PersistentCacheCreateObject, } from '../../src/types'; import { denomToHumanReadableConversion } from './conversion-helpers'; @@ -929,3 +930,15 @@ export const defaultLeaderboardPnlOneDayToUpsert: LeaderboardPnlCreateObject = { currentEquity: '1000', rank: 1, }; + +// ============== Persistent cache Data ============== + +export const defaultKV: PersistentCacheCreateObject = { + key: 'someKey', + value: 'someValue', +}; + +export const defaultKV2: PersistentCacheCreateObject = { + key: 'otherKey', + value: 'otherValue', +}; diff --git a/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts b/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts new file mode 100644 index 0000000000..f0d70073ba --- /dev/null +++ b/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts @@ -0,0 +1,72 @@ +import { PersistentCacheFromDatabase } from '../../src/types'; +import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; +import { defaultKV, defaultKV2 } from '../helpers/constants'; +import * as PersistentCacheTable from '../../src/stores/persistent-cache-table'; + +describe('Persistent cache store', () => { + beforeAll(async () => { + await migrate(); + }); + + afterEach(async () => { + await clearData(); + }); + + afterAll(async () => { + await teardown(); + }); + + it('Successfully creates a key value pair', async () => { + await PersistentCacheTable.create(defaultKV); + }); + + it('Successfully upserts a kv pair multiple times', async () => { + const newKv = { + ...defaultKV, + value: 'someOtherValue', + }; + await PersistentCacheTable.upsert(newKv); + let kv: PersistentCacheFromDatabase | undefined = await PersistentCacheTable.findById( + defaultKV.key, + ); + expect(kv).toEqual(expect.objectContaining(newKv)); + + const newKv2 = { + ...defaultKV, + value: 'someOtherValue2', + }; + await PersistentCacheTable.upsert(newKv2); + kv = await PersistentCacheTable.findById(defaultKV.key); + + expect(kv).toEqual(expect.objectContaining(newKv2)); + }); + + it('Successfully finds all kv pairs', async () => { + await Promise.all([ + PersistentCacheTable.create(defaultKV), + PersistentCacheTable.create(defaultKV2), + ]); + + const kvs: PersistentCacheFromDatabase[] = await PersistentCacheTable.findAll( + {}, + [], + { readReplica: true }, + ); + + expect(kvs.length).toEqual(2); + expect(kvs).toEqual(expect.arrayContaining([ + expect.objectContaining(defaultKV), + expect.objectContaining(defaultKV2) + ])); + }); + + it('Successfully finds a Wallet', async () => { + await PersistentCacheTable.create(defaultKV); + + const kv: PersistentCacheFromDatabase | undefined = await PersistentCacheTable.findById( + defaultKV.key, + ); + + expect(kv).toEqual(expect.objectContaining(defaultKV)); + }); +}); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts new file mode 100644 index 0000000000..2e01a53ce7 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts @@ -0,0 +1,16 @@ +import * as Knex from "knex"; + + +export async function up(knex: Knex): Promise { + return knex.schema.createTable('persistent_cache', (table) => { + table.string('key').primary().notNullable(); + table.text('value').notNullable(); + }); +} + + +export async function down(knex: Knex): Promise { + return knex.schema.dropTable('persistent_cache'); +} + + diff --git a/indexer/packages/postgres/src/helpers/db-helpers.ts b/indexer/packages/postgres/src/helpers/db-helpers.ts index 65949a5407..97aadd9753 100644 --- a/indexer/packages/postgres/src/helpers/db-helpers.ts +++ b/indexer/packages/postgres/src/helpers/db-helpers.ts @@ -28,6 +28,7 @@ const layer1Tables = [ 'trading_rewards', 'trading_reward_aggregations', 'compliance_status', + 'persistent_cache', ]; /** diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index 5ea7775956..34979ef0c6 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -18,6 +18,7 @@ export { default as TradingRewardModel } from './models/trading-reward-model'; export { default as TradingRewardAggregationModel } from './models/trading-reward-aggregation-model'; export { default as SubaccountUsernamesModel } from './models/subaccount-usernames-model'; export { default as LeaderboardPnlModel } from './models/leaderboard-pnl-model'; +export { default as PersistentCacheModel } from './models/persistent-cache-model'; export * as AssetTable from './stores/asset-table'; export * as AssetPositionTable from './stores/asset-position-table'; @@ -43,6 +44,7 @@ export * as TradingRewardTable from './stores/trading-reward-table'; export * as TradingRewardAggregationTable from './stores/trading-reward-aggregation-table'; export * as LeaderboardPnlTable from './stores/leaderboard-pnl-table'; export * as SubaccountUsernamesTable from './stores/subaccount-usernames-table'; +export * as PersistentCacheTable from './stores/persistent-cache-table'; export * as perpetualMarketRefresher from './loops/perpetual-market-refresher'; export * as assetRefresher from './loops/asset-refresher'; diff --git a/indexer/packages/postgres/src/models/persistent-cache-model.ts b/indexer/packages/postgres/src/models/persistent-cache-model.ts new file mode 100644 index 0000000000..20ef74b2c5 --- /dev/null +++ b/indexer/packages/postgres/src/models/persistent-cache-model.ts @@ -0,0 +1,31 @@ +import UpsertQueryBuilder from '../query-builders/upsert'; +import BaseModel from './base-model'; + +export default class PersistentCacheModel extends BaseModel { + static get tableName() { + return 'persistent_cache'; + } + + static get idColumn() { + return 'key'; + } + + static relationMappings = {}; + + static get jsonSchema() { + return { + type: 'object', + required: ['key', 'value'], + properties: { + key: { type: 'string' }, + value: { type: 'string' }, + }, + }; + } + + QueryBuilderType!: UpsertQueryBuilder; + + key!: string; + + value!: string; +} \ No newline at end of file diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index 0436578514..263698e6e7 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -270,6 +270,11 @@ export interface LeaderboardPnlFromDatabase { rank: number; } +export interface PersistentCacheFromDatabase { + key: string; + value: string; +} + export type SubaccountAssetNetTransferMap = { [subaccountId: string]: { [assetId: string]: string } }; export type SubaccountToPerpetualPositionsMap = { [subaccountId: string]: diff --git a/indexer/packages/postgres/src/types/index.ts b/indexer/packages/postgres/src/types/index.ts index 53f1b3c397..3271c86fce 100644 --- a/indexer/packages/postgres/src/types/index.ts +++ b/indexer/packages/postgres/src/types/index.ts @@ -28,4 +28,5 @@ export * from './trading-reward-aggregation-types'; export * from './pagination-types'; export * from './subaccount-usernames-types'; export * from './leaderboard-pnl-types'; +export * from './persistent-cache-types'; export { PositionSide } from './position-types'; diff --git a/indexer/packages/postgres/src/types/persistent-cache-types.ts b/indexer/packages/postgres/src/types/persistent-cache-types.ts new file mode 100644 index 0000000000..36f324fe01 --- /dev/null +++ b/indexer/packages/postgres/src/types/persistent-cache-types.ts @@ -0,0 +1,14 @@ +export interface PersistentCacheCreateObject { + key: string, + value: string, +} + +export interface PersistentCacheUpdateObject { + key: string, + value: string, +} + +export enum PersistentCacheColumns { + key = 'key', + value = 'value', +} diff --git a/indexer/packages/postgres/src/types/query-types.ts b/indexer/packages/postgres/src/types/query-types.ts index 75417a6d8b..733e754d59 100644 --- a/indexer/packages/postgres/src/types/query-types.ts +++ b/indexer/packages/postgres/src/types/query-types.ts @@ -88,6 +88,7 @@ export enum QueryableField { USERNAME = 'username', TIMESPAN = 'timeSpan', RANK = 'rank', + KEY = 'key', } export interface QueryConfig { @@ -321,3 +322,7 @@ export interface LeaderboardPnlQueryConfig extends QueryConfig { [QueryableField.TIMESPAN]?: string[]; [QueryableField.RANK]?: number[]; } + +export interface PersistentCacheQueryConfig extends QueryConfig { + [QueryableField.KEY]?: string; +} From 205eda7e50fbfdd89639bfbba98dabcf6a8341bb Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 29 Aug 2024 17:37:01 -0400 Subject: [PATCH 2/3] add table file --- .../src/stores/persistent-cache-table.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 indexer/packages/postgres/src/stores/persistent-cache-table.ts diff --git a/indexer/packages/postgres/src/stores/persistent-cache-table.ts b/indexer/packages/postgres/src/stores/persistent-cache-table.ts new file mode 100644 index 0000000000..b7f16e566d --- /dev/null +++ b/indexer/packages/postgres/src/stores/persistent-cache-table.ts @@ -0,0 +1,95 @@ +import { PartialModelObject, QueryBuilder } from 'objection'; + +import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; +import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; +import Transaction from '../helpers/transaction'; +import PersistentCacheModel from '../models/persistent-cache-model'; +import { + Options, + Ordering, + QueryableField, + QueryConfig, + PersistentCacheColumns, + PersistentCacheCreateObject, + PersistentCacheFromDatabase, + PersistentCacheQueryConfig, + PersistentCacheUpdateObject, +} from '../types'; + +export async function findAll( + { + key, + limit, + }: PersistentCacheQueryConfig, + requiredFields: QueryableField[], + options: Options = DEFAULT_POSTGRES_OPTIONS, +): Promise { + verifyAllRequiredFields( + { + key, + limit, + } as QueryConfig, + requiredFields, + ); + + let baseQuery: QueryBuilder = setupBaseQuery( + PersistentCacheModel, + options, + ); + + if (key) { + baseQuery = baseQuery.where(PersistentCacheColumns.key, key); + } + + if (options.orderBy !== undefined) { + for (const [column, order] of options.orderBy) { + baseQuery = baseQuery.orderBy( + column, + order, + ); + } + } else { + baseQuery = baseQuery.orderBy( + PersistentCacheColumns.key, + Ordering.ASC, + ); + } + + if (limit) { + baseQuery = baseQuery.limit(limit); + } + + return baseQuery.returning('*'); +} + +export async function create( + kvToCreate: PersistentCacheCreateObject, + options: Options = { txId: undefined }, +): Promise { + return PersistentCacheModel.query( + Transaction.get(options.txId), + ).insert(kvToCreate).returning('*'); +} + +export async function upsert( + kvToUpsert: PersistentCacheCreateObject, + options: Options = { txId: undefined }, +): Promise { + const kvs: PersistentCacheModel[] = await PersistentCacheModel.query( + Transaction.get(options.txId), + ).upsert(kvToUpsert).returning('*'); + // should only ever be one key value + return kvs[0]; +} +export async function findById( + kv: string, + options: Options = DEFAULT_POSTGRES_OPTIONS, +): Promise { + const baseQuery: QueryBuilder = setupBaseQuery( + PersistentCacheModel, + options, + ); + return baseQuery + .findById(kv) + .returning('*'); +} From ef21ed436fcc2d4636cdb61b9d7c9fabfe5102c3 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 29 Aug 2024 17:48:00 -0400 Subject: [PATCH 3/3] linting fixes --- .../__tests__/stores/persistent-cache-table.test.ts | 6 +++--- ...ts => 20240829171730_create_persistent_cache_table.ts} | 8 ++------ .../postgres/src/models/persistent-cache-model.ts | 2 +- .../postgres/src/stores/persistent-cache-table.ts | 5 ++--- indexer/packages/postgres/src/types/db-model-types.ts | 4 ++-- .../packages/postgres/src/types/persistent-cache-types.ts | 5 ----- indexer/packages/postgres/src/types/query-types.ts | 2 +- 7 files changed, 11 insertions(+), 21 deletions(-) rename indexer/packages/postgres/src/db/migrations/migration_files/{20240828130730_create_persistent_cache_table.ts => 20240829171730_create_persistent_cache_table.ts} (79%) diff --git a/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts b/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts index f0d70073ba..02a926dd04 100644 --- a/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/persistent-cache-table.test.ts @@ -30,7 +30,7 @@ describe('Persistent cache store', () => { defaultKV.key, ); expect(kv).toEqual(expect.objectContaining(newKv)); - + const newKv2 = { ...defaultKV, value: 'someOtherValue2', @@ -56,11 +56,11 @@ describe('Persistent cache store', () => { expect(kvs.length).toEqual(2); expect(kvs).toEqual(expect.arrayContaining([ expect.objectContaining(defaultKV), - expect.objectContaining(defaultKV2) + expect.objectContaining(defaultKV2), ])); }); - it('Successfully finds a Wallet', async () => { + it('Successfully finds a kv pair', async () => { await PersistentCacheTable.create(defaultKV); const kv: PersistentCacheFromDatabase | undefined = await PersistentCacheTable.findById( diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240829171730_create_persistent_cache_table.ts similarity index 79% rename from indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts rename to indexer/packages/postgres/src/db/migrations/migration_files/20240829171730_create_persistent_cache_table.ts index 2e01a53ce7..0bd053a82a 100644 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240828130730_create_persistent_cache_table.ts +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240829171730_create_persistent_cache_table.ts @@ -1,16 +1,12 @@ -import * as Knex from "knex"; - +import * as Knex from 'knex'; export async function up(knex: Knex): Promise { return knex.schema.createTable('persistent_cache', (table) => { table.string('key').primary().notNullable(); - table.text('value').notNullable(); + table.string('value').notNullable(); }); } - export async function down(knex: Knex): Promise { return knex.schema.dropTable('persistent_cache'); } - - diff --git a/indexer/packages/postgres/src/models/persistent-cache-model.ts b/indexer/packages/postgres/src/models/persistent-cache-model.ts index 20ef74b2c5..cd4c3480bc 100644 --- a/indexer/packages/postgres/src/models/persistent-cache-model.ts +++ b/indexer/packages/postgres/src/models/persistent-cache-model.ts @@ -28,4 +28,4 @@ export default class PersistentCacheModel extends BaseModel { key!: string; value!: string; -} \ No newline at end of file +} diff --git a/indexer/packages/postgres/src/stores/persistent-cache-table.ts b/indexer/packages/postgres/src/stores/persistent-cache-table.ts index b7f16e566d..744d4b8334 100644 --- a/indexer/packages/postgres/src/stores/persistent-cache-table.ts +++ b/indexer/packages/postgres/src/stores/persistent-cache-table.ts @@ -1,4 +1,4 @@ -import { PartialModelObject, QueryBuilder } from 'objection'; +import { QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; @@ -13,7 +13,6 @@ import { PersistentCacheCreateObject, PersistentCacheFromDatabase, PersistentCacheQueryConfig, - PersistentCacheUpdateObject, } from '../types'; export async function findAll( @@ -78,7 +77,7 @@ export async function upsert( const kvs: PersistentCacheModel[] = await PersistentCacheModel.query( Transaction.get(options.txId), ).upsert(kvToUpsert).returning('*'); - // should only ever be one key value + // should only ever be one key value pair return kvs[0]; } export async function findById( diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index e2fb15a457..739344650f 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -274,8 +274,8 @@ export interface LeaderboardPnlFromDatabase { } export interface PersistentCacheFromDatabase { - key: string; - value: string; + key: string, + value: string, } export type SubaccountAssetNetTransferMap = { [subaccountId: string]: diff --git a/indexer/packages/postgres/src/types/persistent-cache-types.ts b/indexer/packages/postgres/src/types/persistent-cache-types.ts index 36f324fe01..6022e6764d 100644 --- a/indexer/packages/postgres/src/types/persistent-cache-types.ts +++ b/indexer/packages/postgres/src/types/persistent-cache-types.ts @@ -3,11 +3,6 @@ export interface PersistentCacheCreateObject { value: string, } -export interface PersistentCacheUpdateObject { - key: string, - value: string, -} - export enum PersistentCacheColumns { key = 'key', value = 'value', diff --git a/indexer/packages/postgres/src/types/query-types.ts b/indexer/packages/postgres/src/types/query-types.ts index 391ccc52d8..fae4976849 100644 --- a/indexer/packages/postgres/src/types/query-types.ts +++ b/indexer/packages/postgres/src/types/query-types.ts @@ -326,5 +326,5 @@ export interface LeaderboardPnlQueryConfig extends QueryConfig { } export interface PersistentCacheQueryConfig extends QueryConfig { - [QueryableField.KEY]?: string; + [QueryableField.KEY]?: string, }