Merge branch 'main' of https://github.com/dydxprotocol/v4-chain into …

…wl/rm_extra_lookup

indexer/Dockerfile.auxo.remote

@@ -6,6 +6,9 @@ WORKDIR /usr/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy auxo and imported packages
 COPY ./packages/base/ ./packages/base/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
@@ -19,7 +22,7 @@ COPY tsconfig.json ./
 RUN npm install -g pnpm@6
 
 # Install npm modules using pnpm
-RUN pnpm install --loglevel warn --frozen-lockfile
+RUN pnpm install --loglevel warn --frozen-lockfile --unsafe-perm
 RUN pnpm run build:prod:all
 
 FROM public.ecr.aws/lambda/nodejs:16

indexer/Dockerfile.bazooka.remote

@@ -4,7 +4,10 @@ FROM public.ecr.aws/lambda/nodejs:16 as builder
 WORKDIR /usr/app
 
 # Copy pnpm lock and workspace and package.json from base directory
-COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
+COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./patches ./
+
+# Copy patches
+COPY ./patches ./patches
 
 # Copy bazooka and imported packages
 COPY ./packages/base/ ./packages/base/
@@ -22,7 +25,7 @@ COPY tsconfig.json ./
 RUN npm install -g pnpm@6
 
 # Install npm modules using pnpm
-RUN pnpm install --loglevel warn --frozen-lockfile
+RUN pnpm install --loglevel warn --frozen-lockfile --unsafe-perm
 RUN pnpm run build:prod:all
 
 FROM public.ecr.aws/lambda/nodejs:16

indexer/Dockerfile.postgres-package.local

@@ -11,6 +11,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from postgres and imported packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/postgres/package.json ./packages/postgres/
@@ -37,7 +40,7 @@ RUN chown dydx -R /home/dydx/app
 USER dydx
 
 # Install npm modules using pnpm
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 WORKDIR /home/dydx/app/packages/postgres
 

indexer/Dockerfile.service.local

@@ -13,6 +13,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from all packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/kafka/package.json ./packages/kafka/
@@ -42,7 +45,7 @@ COPY ./scripts/container-run.sh /home/dydx/app/services/$service/scripts/
 RUN chown dydx -R /home/dydx/app
 
 USER dydx
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 WORKDIR /home/dydx/app/services/$service
 

indexer/Dockerfile.service.remote

@@ -13,6 +13,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from all packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/kafka/package.json ./packages/kafka/
@@ -42,7 +45,7 @@ COPY ./scripts/container-run.sh /home/dydx/app/services/$service/scripts/
 RUN chown dydx -R /home/dydx/app
 
 # Install npm modules using pnpm
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 USER dydx
 

indexer/package.json

@@ -13,8 +13,15 @@
     "lint:all": "pnpm run --parallel lint",
     "lint:fix:all": "pnpm run --parallel lint:fix",
     "coverage:all": "pnpm recursive run --workspace-concurrency=1 coverage",
-    "test:all": "pnpm recursive run --workspace-concurrency=1 test"
+    "test:all": "pnpm recursive run --workspace-concurrency=1 test",
+    "postinstall": "patch-package"
   },
   "author": "",
-  "license": "AGPL-3.0"
+  "license": "AGPL-3.0",
+  "dependencies": {
+    "@milahu/patch-package": "6.4.14"
+  },
+  "devDependencies": {
+    "@types/ws": "8.5.10"
+  }
 }

indexer/packages/postgres/__tests__/stores/pnl-ticks-table.test.ts

@@ -184,14 +184,27 @@ describe('PnlTicks store', () => {
       }),
     ]);
 
-    const latestBlocktime: string = await PnlTicksTable.findLatestProcessedBlocktime();
+    const {
+      maxBlockTime, count,
+    }: {
+      maxBlockTime: string,
+      count: number
+    } = await PnlTicksTable.findLatestProcessedBlocktimeAndCount();
 
-    expect(latestBlocktime).toEqual(blockTime);
+    expect(maxBlockTime).toEqual(blockTime);
+    expect(count).toEqual(2);
   });
 
   it('Successfully finds latest block time without any pnl ticks', async () => {
-    const latestBlocktime: string = await PnlTicksTable.findLatestProcessedBlocktime();
-    expect(latestBlocktime).toEqual(ZERO_TIME_ISO_8601);
+    const {
+      maxBlockTime, count,
+    }: {
+      maxBlockTime: string,
+      count: number
+    } = await PnlTicksTable.findLatestProcessedBlocktimeAndCount();
+
+    expect(maxBlockTime).toEqual(ZERO_TIME_ISO_8601);
+    expect(count).toEqual(0);
   });
 
   it('createMany PnlTicks, find most recent pnl ticks for each account', async () => {

indexer/packages/postgres/src/stores/pnl-ticks-table.ts

@@ -212,17 +212,24 @@ function convertPnlTicksFromDatabaseToPnlTicksCreateObject(
   return _.omit(pnlTicksFromDatabase, PnlTicksColumns.id);
 }
 
-export async function findLatestProcessedBlocktime(): Promise<string> {
+export async function findLatestProcessedBlocktimeAndCount(): Promise<{
+  maxBlockTime: string,
+  count: number,
+}> {
   const result: {
-    rows: [{ max: string }]
+    rows: [{ max: string, count: number }]
   } = await knexReadReplica.getConnection().raw(
     `
-    SELECT MAX("blockTime")
+    SELECT MAX("blockTime") as max, COUNT(*) as count
     FROM "pnl_ticks"
     `
     ,
-  ) as unknown as { rows: [{ max: string }] };
-  return result.rows[0].max || ZERO_TIME_ISO_8601;
+  ) as unknown as { rows: [{ max: string, count: number }] };
+
+  return {
+    maxBlockTime: result.rows[0].max || ZERO_TIME_ISO_8601,
+    count: Number(result.rows[0].count) || 0,
+  };
 }
 
 export async function findMostRecentPnlTickForEachAccount(

indexer/packages/redis/__tests__/helpers/constants.ts

@@ -17,23 +17,33 @@ import {
   IndexerSubaccountId,
   IndexerOrder_ConditionType,
   OrderRemovalReason,
+  OrderReplaceV1,
 } from '@dydxprotocol-indexer/v4-protos';
 import Long from 'long';
 
 export type OffChainUpdateOrderPlaceUpdateMessage = {
   orderUpdate: undefined,
   orderRemove: undefined,
   orderPlace: OrderPlaceV1,
+  orderReplace: undefined
 };
 export type OffChainUpdateOrderRemoveUpdateMessage = {
   orderUpdate: undefined,
   orderRemove: OrderRemoveV1,
   orderPlace: undefined,
+  orderReplace: undefined
 };
 export type OffChainUpdateOrderUpdateUpdateMessage = {
   orderUpdate: OrderUpdateV1,
   orderRemove: undefined,
   orderPlace: undefined,
+  orderReplace: undefined
+};
+export type OffChainUpdateOrderReplaceUpdateMessage = {
+  orderUpdate: undefined,
+  orderRemove: undefined,
+  orderPlace: undefined,
+  orderReplace: OrderReplaceV1,
 };
 
 export const defaultSubaccountId: IndexerSubaccountId = {
@@ -58,6 +68,18 @@ export const defaultOrderIdConditional: IndexerOrderId = {
   clobPairId: parseInt(testConstants.defaultPerpetualMarket.clobPairId, 10),
   orderFlags: ORDER_FLAG_CONDITIONAL,
 };
+export const defaultReplacementOrderId: IndexerOrderId = {
+  subaccountId: defaultSubaccountId,
+  clientId: 4,
+  clobPairId: parseInt(testConstants.defaultPerpetualMarket.clobPairId, 10),
+  orderFlags: ORDER_FLAG_SHORT_TERM,
+};
+export const defaultReplacementOrderIdGTBT: IndexerOrderId = {
+  subaccountId: defaultSubaccountId,
+  clientId: 5,
+  clobPairId: parseInt(testConstants.defaultPerpetualMarket.clobPairId, 10),
+  orderFlags: ORDER_FLAG_LONG_TERM,
+};
 
 export const defaultSubaccountUuid: string = SubaccountTable.uuid(
   defaultSubaccountId.owner,
@@ -79,12 +101,24 @@ export const defaultOrder: IndexerOrder = {
   conditionType: IndexerOrder_ConditionType.CONDITION_TYPE_UNSPECIFIED,
   conditionalOrderTriggerSubticks: Long.fromValue(0, true),
 };
+export const defaultReplacementOrder: IndexerOrder = {
+  ...defaultOrder,
+  orderId: defaultReplacementOrderId,
+  quantums: Long.fromValue(1_500_000, true),
+  goodTilBlock: 1160,
+};
 export const defaultOrderGoodTilBlockTime: IndexerOrder = {
   ...defaultOrder,
   orderId: defaultOrderIdGoodTilBlockTime,
   goodTilBlockTime: 1_200_000_000,
   goodTilBlock: undefined,
 };
+export const defaultReplacementOrderGTBT: IndexerOrder = {
+  ...defaultOrder,
+  orderId: defaultReplacementOrderIdGTBT,
+  goodTilBlockTime: 1_300_000_000,
+  goodTilBlock: undefined,
+};
 export const defaultConditionalOrder: IndexerOrder = {
   ...defaultOrderGoodTilBlockTime,
   orderId: defaultOrderIdConditional,
@@ -107,6 +141,12 @@ export const defaultOrderUuidGoodTilBlockTime: string = OrderTable.orderIdToUuid
 export const defaultOrderUuidConditional: string = OrderTable.orderIdToUuid(
   defaultOrderIdConditional,
 );
+export const defaultReplacementOrderUuid: string = OrderTable.orderIdToUuid(
+  defaultReplacementOrderId,
+);
+export const defaultReplacementOrderUuidGTBT: string = OrderTable.orderIdToUuid(
+  defaultReplacementOrderIdGTBT,
+);
 
 export const defaultPrice = protocolTranslations.subticksToPrice(
   defaultOrder.subticks.toString(),
@@ -150,6 +190,7 @@ export const orderPlace: OffChainUpdateOrderPlaceUpdateMessage = {
     order: defaultOrder,
     placementStatus: OrderPlaceV1_OrderPlacementStatus.ORDER_PLACEMENT_STATUS_BEST_EFFORT_OPENED,
   },
+  orderReplace: undefined,
 };
 export const orderRemove: OffChainUpdateOrderRemoveUpdateMessage = {
   orderPlace: undefined,
@@ -159,6 +200,7 @@ export const orderRemove: OffChainUpdateOrderRemoveUpdateMessage = {
     reason: OrderRemovalReason.ORDER_REMOVAL_REASON_INTERNAL_ERROR,
     removalStatus: OrderRemoveV1_OrderRemovalStatus.ORDER_REMOVAL_STATUS_BEST_EFFORT_CANCELED,
   },
+  orderReplace: undefined,
 };
 export const orderUpdate: OffChainUpdateOrderUpdateUpdateMessage = {
   orderPlace: undefined,
@@ -167,6 +209,17 @@ export const orderUpdate: OffChainUpdateOrderUpdateUpdateMessage = {
     orderId: defaultOrderId,
     totalFilledQuantums: Long.fromValue(250_500, true),
   },
+  orderReplace: undefined,
+};
+export const orderReplace: OffChainUpdateOrderReplaceUpdateMessage = {
+  orderPlace: undefined,
+  orderRemove: undefined,
+  orderUpdate: undefined,
+  orderReplace: {
+    oldOrderId: defaultOrderId,
+    order: defaultOrder,
+    placementStatus: OrderPlaceV1_OrderPlacementStatus.ORDER_PLACEMENT_STATUS_BEST_EFFORT_OPENED,
+  },
 };
 
 export const isolatedSubaccountId: IndexerSubaccountId = {

indexer/packages/v4-protos/src/codegen/dydxprotocol/vault/query.ts

@@ -37,17 +37,17 @@ export interface QueryVaultRequestSDKType {
 export interface QueryVaultResponse {
   vaultId?: VaultId;
   subaccountId?: SubaccountId;
-  equity: Long;
-  inventory: Long;
+  equity: Uint8Array;
+  inventory: Uint8Array;
   totalShares: Long;
 }
 /** QueryVaultResponse is a response type for the Vault RPC method. */
 
 export interface QueryVaultResponseSDKType {
   vault_id?: VaultIdSDKType;
   subaccount_id?: SubaccountIdSDKType;
-  equity: Long;
-  inventory: Long;
+  equity: Uint8Array;
+  inventory: Uint8Array;
   total_shares: Long;
 }
 /** QueryAllVaultsRequest is a request type for the AllVaults RPC method. */
@@ -249,8 +249,8 @@ function createBaseQueryVaultResponse(): QueryVaultResponse {
   return {
     vaultId: undefined,
     subaccountId: undefined,
-    equity: Long.UZERO,
-    inventory: Long.UZERO,
+    equity: new Uint8Array(),
+    inventory: new Uint8Array(),
     totalShares: Long.UZERO
   };
 }
@@ -265,12 +265,12 @@ export const QueryVaultResponse = {
       SubaccountId.encode(message.subaccountId, writer.uint32(18).fork()).ldelim();
     }
 
-    if (!message.equity.isZero()) {
-      writer.uint32(24).uint64(message.equity);
+    if (message.equity.length !== 0) {
+      writer.uint32(26).bytes(message.equity);
     }
 
-    if (!message.inventory.isZero()) {
-      writer.uint32(32).uint64(message.inventory);
+    if (message.inventory.length !== 0) {
+      writer.uint32(34).bytes(message.inventory);
     }
 
     if (!message.totalShares.isZero()) {
@@ -298,11 +298,11 @@ export const QueryVaultResponse = {
           break;
 
         case 3:
-          message.equity = (reader.uint64() as Long);
+          message.equity = reader.bytes();
           break;
 
         case 4:
-          message.inventory = (reader.uint64() as Long);
+          message.inventory = reader.bytes();
           break;
 
         case 5:
@@ -322,8 +322,8 @@ export const QueryVaultResponse = {
     const message = createBaseQueryVaultResponse();
     message.vaultId = object.vaultId !== undefined && object.vaultId !== null ? VaultId.fromPartial(object.vaultId) : undefined;
     message.subaccountId = object.subaccountId !== undefined && object.subaccountId !== null ? SubaccountId.fromPartial(object.subaccountId) : undefined;
-    message.equity = object.equity !== undefined && object.equity !== null ? Long.fromValue(object.equity) : Long.UZERO;
-    message.inventory = object.inventory !== undefined && object.inventory !== null ? Long.fromValue(object.inventory) : Long.UZERO;
+    message.equity = object.equity ?? new Uint8Array();
+    message.inventory = object.inventory ?? new Uint8Array();
     message.totalShares = object.totalShares !== undefined && object.totalShares !== null ? Long.fromValue(object.totalShares) : Long.UZERO;
     return message;
   }

indexer/patches/@types+ws+8.5.10.patch

@@ -0,0 +1,22 @@
+# generated by patch-package 6.4.14
+#
+# declared package:
+#   @types/ws: 8.5.10
+#
+diff --git a/node_modules/@types/ws/index.d.ts b/node_modules/@types/ws/index.d.ts
+index 94d1dca..d9abf0b 100644
+--- a/node_modules/@types/ws/index.d.ts
++++ b/node_modules/@types/ws/index.d.ts
+@@ -318,7 +318,11 @@ declare namespace WebSocket {
+         host?: string | undefined;
+         port?: number | undefined;
+         backlog?: number | undefined;
+-        server?: HTTPServer<V> | HTTPSServer<V> | undefined;
++        // LOCAL DYDX PATCH
++        allowSynchronousEvents?: boolean;
++        autoPong?: boolean;
++        server?: HTTPServer | HTTPSServer | undefined;
++        // END LOCAL DYDX PATCH
+         verifyClient?:
+             | VerifyClientCallbackAsync<InstanceType<V>>
+             | VerifyClientCallbackSync<InstanceType<V>>