self review

packages/store-indexer/src/postgres/getLogs.ts

@@ -2,13 +2,12 @@ import { Sql } from "postgres";
 import { Middleware } from "koa";
 import Router from "@koa/router";
 import compose from "koa-compose";
-import { input } from "@latticexyz/store-sync/trpc-indexer";
+import { input } from "@latticexyz/store-sync/indexer-client";
 import { storeTables } from "@latticexyz/store-sync";
 import { queryLogs } from "./queryLogs";
 import { recordToLog } from "./recordToLog";
 import { debug } from "../debug";
 import { createBenchmark } from "@latticexyz/common";
-import superjson from "superjson";
 import { compress } from "../compress";
 
 export function getLogs(database: Sql): Middleware {
@@ -28,12 +27,10 @@ export function getLogs(database: Sql): Middleware {
       const logs = records.map(recordToLog);
       benchmark("map records to logs");
 
-      const blockNumber = BigInt(records[0].chainBlockNumber);
+      const blockNumber = records[0].chainBlockNumber;
 
+      ctx.body = JSON.stringify({ blockNumber, logs });
       ctx.status = 200;
-
-      // TODO: replace superjson with more efficient encoding
-      ctx.body = superjson.stringify({ blockNumber, logs });
     } catch (error) {
       ctx.status = 500;
       ctx.body = error;

packages/store-indexer/src/postgres/queryLogs.ts

@@ -2,7 +2,7 @@ import { isNotNull } from "@latticexyz/common/utils";
 import { PendingQuery, Row, Sql } from "postgres";
 import { hexToBytes } from "viem";
 import { z } from "zod";
-import { input } from "@latticexyz/store-sync/trpc-indexer";
+import { input } from "@latticexyz/store-sync/indexer-client";
 import { Record } from "./types";
 
 function and(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {

packages/store-indexer/src/postgres/types.ts

@@ -7,13 +7,13 @@ export type RecordData = {
   staticData: Hex | null;
   encodedLengths: Hex | null;
   dynamicData: Hex | null;
-  lastUpdatedBlockNumber: bigint;
+  lastUpdatedBlockNumber: string;
 };
 
 export type RecordMetadata = {
   indexerVersion: string;
   chainId: string;
-  chainBlockNumber: bigint;
+  chainBlockNumber: string;
   totalRows: number;
 };
 

packages/store-sync/package.json

@@ -11,6 +11,7 @@
   "type": "module",
   "exports": {
     ".": "./dist/index.js",
+    "./indexer-client": "./dist/indexer-client/index.js",
     "./postgres": "./dist/postgres/index.js",
     "./postgres-decoded": "./dist/postgres-decoded/index.js",
     "./recs": "./dist/recs/index.js",
@@ -23,6 +24,9 @@
       "index": [
         "./src/index.ts"
       ],
+      "indexer-client": [
+        "./src/indexer-client/index.ts"
+      ],
       "postgres": [
         "./src/postgres/index.ts"
       ],

packages/store-sync/src/getSnapshot.ts

@@ -1,8 +1,8 @@
 import { StorageAdapterBlock, SyncOptions } from "./common";
 import { debug as parentDebug } from "./debug";
-import { createIndexerClient as createTrpcIndexerClient } from "./trpc-indexer";
 import { TRPCClientError } from "@trpc/client";
 import { tablesWithRecordsToLogs } from "./tablesWithRecordsToLogs";
+import { createIndexerClient as createTrpcIndexerClient } from "./trpc-indexer";
 import { createIndexerClient } from "./indexer-client";
 
 const debug = parentDebug.extend("getSnapshot");
@@ -40,9 +40,9 @@ export async function getSnapshot({
 
   if (!indexerUrl) return;
 
-  const url = new URL(indexerUrl);
-  const indexer = createIndexerClient({ url: url.origin });
-  const trpcIndexer = createTrpcIndexerClient({ url: `${url.origin}/trpc` });
+  const indexerOrigin = new URL(indexerUrl).origin;
+  const indexer = createIndexerClient({ url: indexerOrigin });
+  const trpcIndexer = createTrpcIndexerClient({ url: `${indexerOrigin}/trpc` });
 
   try {
     debug("fetching logs from indexer via get", indexerUrl);

packages/store-sync/src/indexer-client/createIndexerClient.ts

@@ -1,7 +1,6 @@
 import { z } from "zod";
 import { input } from "./input";
 import { StorageAdapterBlock } from "../common";
-import superjson from "superjson";
 
 type CreateIndexerClientOptions = {
   /**
@@ -27,16 +26,16 @@ export function createIndexerClient({ url }: CreateIndexerClientOptions): Indexe
       const response = await fetch(`${url}/get/logs?input=${input}`, { method: "GET" });
 
       // TODO: could we return a readable stream here instead of fetching the entire response right away?
-      const result = superjson.parse(await response.text());
+      const result = await response.json();
       if (!isStorageAdapterBlock(result)) {
-        throw new Error("Unexpected response:\n" + superjson.stringify(result));
+        throw new Error("Unexpected response:\n" + JSON.stringify(result));
       }
 
-      return result;
+      return { ...result, blockNumber: BigInt(result.blockNumber) };
     },
   };
 }
 
-function isStorageAdapterBlock(data: any): data is StorageAdapterBlock {
-  return data && typeof data.blockNumber === "bigint" && Array.isArray(data.logs);
+function isStorageAdapterBlock(data: any): data is Omit<StorageAdapterBlock, "blockNumber"> & { blockNumber: string } {
+  return data && typeof data.blockNumber === "string" && Array.isArray(data.logs);
 }

packages/store-sync/src/indexer-client/index.ts

@@ -1 +1,2 @@
 export * from "./createIndexerClient";
+export * from "./input";

packages/store-sync/src/trpc-indexer/createAppRouter.ts

@@ -1,22 +1,7 @@
-import { z } from "zod";
 import { QueryAdapter } from "./common";
-import { isHex } from "viem";
 import { initTRPC } from "@trpc/server";
 import superjson from "superjson";
-
-export const input = z.object({
-  chainId: z.number(),
-  address: z.string().refine(isHex).optional(),
-  filters: z
-    .array(
-      z.object({
-        tableId: z.string().refine(isHex),
-        key0: z.string().refine(isHex).optional(),
-        key1: z.string().refine(isHex).optional(),
-      })
-    )
-    .default([]),
-});
+import { input } from "../indexer-client/input";
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export function createAppRouter() {

packages/store-sync/tsup.config.ts

@@ -8,6 +8,7 @@ export default defineConfig({
     "src/postgres-decoded/index.ts",
     "src/recs/index.ts",
     "src/trpc-indexer/index.ts",
+    "src/indexer-client/index.ts",
     "src/zustand/index.ts",
   ],
   target: "esnext",