feat(store-indexer, store-sync): improve query performance and enable…

… compression, add new api (#2026) Co-authored-by: Kevin Ingersoll <[email protected]>
latticexyz · Dec 7, 2023 · 4c1dcd8 · 4c1dcd8
1 parent ffcf64e
commit 4c1dcd8
Show file tree

Hide file tree

Showing 33 changed files with 544 additions and 83 deletions.
diff --git a/.changeset/tough-moose-pay.md b/.changeset/tough-moose-pay.md
@@ -0,0 +1,7 @@
+---
+"@latticexyz/common": minor
+---
+
+- Added a `Result<Ok, Err>` type for more explicit and typesafe error handling ([inspired by Rust](https://doc.rust-lang.org/std/result/)).
+
+- Added a `includes` util as typesafe alternative to [`Array.prototype.includes()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/includes).
diff --git a/.changeset/wild-years-search.md b/.changeset/wild-years-search.md
@@ -0,0 +1,22 @@
+---
+"@latticexyz/store-indexer": minor
+"@latticexyz/store-sync": minor
+---
+
+- Improved query performance by 10x by moving from drizzle ORM to handcrafted SQL.
+- Moved away from `trpc` for more granular control over the transport layer.
+  Added an `/api/logs` endpoint using the new query and gzip compression for 40x less data transferred over the wire.
+  Deprecated the `/trpc/getLogs` and `/trpc/findAll` endpoints.
+- Added a `createIndexerClient` client for the new `/api` indexer API exported from `@latticexyz/store-sync/indexer-client`.
+  The `createIndexerClient` export from `@latticexyz/store-sync/trpc-indexer` is deprecated.
+
+```diff
+- import { createIndexerClient } from "@latticexyz/store-sync/trpc-indexer";
++ import { createIndexerClient } from "@latticexyz/store-sync/indexer-client";
+
+- const indexer = createIndexerClient({ url: "https://indexer.holesky.redstone.xyz/trpc" });
++ const indexer = createIndexerClient({ url: "https://indexer.holesky.redstone.xyz" });
+
+- const snapshot = indexer.getLogs.query(options);
++ const snapshot = indexer.getLogs(options);
+```
diff --git a/e2e/packages/contracts/worlds.json b/e2e/packages/contracts/worlds.json
@@ -1,5 +1,5 @@
 {
   "31337": {
-    "address": "0x6e9474e9c83676b9a71133ff96db43e7aa0a4342"
+    "address": "0x2ea123a56f2e986c9844bf4dc13050c4df200b29"
   }
 }
diff --git a/e2e/packages/sync-test/indexerSync.test.ts b/e2e/packages/sync-test/indexerSync.test.ts
@@ -23,7 +23,7 @@ import { waitForInitialSync } from "./data/waitForInitialSync";
 
 const env = z
   .object({
-    DATABASE_URL: z.string().default("postgres://127.0.0.1/postgres"),
+    DATABASE_URL: z.string().default("postgres://127.0.0.1/postgres_e2e"),
   })
   .parse(process.env, {
     errorMap: (issue) => ({
@@ -57,18 +57,20 @@ describe("Sync from indexer", async () => {
     await openClientWithRootAccount(page, { indexerUrl: `http://127.0.0.1:9999/trpc` });
     await waitForInitialSync(page);
 
-    expect(asyncErrorHandler.getErrors()).toHaveLength(1);
-    expect(asyncErrorHandler.getErrors()[0]).toContain("error getting snapshot");
+    const errors = asyncErrorHandler.getErrors();
+    expect(errors).toHaveLength(2);
+    expect(errors[0]).toContain("Failed to fetch");
+    expect(errors[1]).toContain("error getting snapshot");
   });
 
   describe.each([["sqlite"], ["postgres"]] as const)("%s indexer", (indexerType) => {
     let indexerIteration = 1;
-    let indexer: ReturnType<typeof startIndexer>;
+    let indexer: Awaited<ReturnType<typeof startIndexer>>;
 
     beforeEach(async () => {
       // Start indexer
       const port = 3000 + indexerIteration++;
-      indexer = startIndexer({
+      indexer = await startIndexer({
         port,
         rpcHttpUrl,
         reportError: asyncErrorHandler.reportError,

diff --git a/e2e/packages/sync-test/package.json b/e2e/packages/sync-test/package.json
@@ -4,8 +4,9 @@
   "private": true,
   "license": "MIT",
   "scripts": {
-    "test": "vitest --run",
-    "test:ci": "pnpm run test"
+    "setup": "psql postgres://127.0.0.1/postgres -c \"CREATE DATABASE postgres_e2e;\"",
+    "test": "DATABASE_URL=postgres://127.0.0.1/postgres_e2e vitest --run",
+    "test:ci": "vitest --run"
   },
   "devDependencies": {
     "@latticexyz/cli": "link:../../../packages/cli",
@@ -20,8 +21,10 @@
     "abitype": "0.9.8",
     "chalk": "^5.2.0",
     "dotenv": "^16.0.3",
+    "drizzle-orm": "^0.28.5",
     "execa": "^7.1.1",
     "happy-dom": "^12.10.3",
+    "postgres": "3.3.5",
     "typescript": "5.1.6",
     "viem": "1.14.0",
     "vite": "^4.2.1",

diff --git a/e2e/packages/sync-test/setup/startIndexer.ts b/e2e/packages/sync-test/setup/startIndexer.ts
@@ -1,6 +1,9 @@
 import chalk from "chalk";
 import { execa } from "execa";
 import { rmSync } from "node:fs";
+import { cleanDatabase } from "@latticexyz/store-sync/postgres";
+import { drizzle } from "drizzle-orm/postgres-js";
+import postgres from "postgres";
 import path from "node:path";
 
 type IndexerOptions =
@@ -19,7 +22,7 @@ type StartIndexerOptions = {
   reportError: (error: string) => void;
 } & IndexerOptions;
 
-export function startIndexer(opts: StartIndexerOptions) {
+export async function startIndexer(opts: StartIndexerOptions) {
   let resolve: () => void;
   let reject: (reason?: string) => void;
   const doneSyncing = new Promise<void>((res, rej) => {
@@ -37,6 +40,9 @@ export function startIndexer(opts: StartIndexerOptions) {
   };
   console.log(chalk.magenta("[indexer]:"), "starting indexer", env);
 
+  // Clean the test db
+  await cleanUp();
+
   const proc = execa("pnpm", opts.indexer === "postgres" ? ["start:postgres"] : ["start:sqlite"], {
     cwd: path.join(__dirname, "..", "..", "..", "..", "packages", "store-indexer"),
     env,
@@ -67,13 +73,22 @@ export function startIndexer(opts: StartIndexerOptions) {
   proc.stdout?.on("data", (data) => onLog(data.toString()));
   proc.stderr?.on("data", (data) => onLog(data.toString()));
 
-  function cleanUp() {
+  async function cleanUp() {
     // attempt to clean up sqlite file
     if (opts.indexer === "sqlite") {
       try {
         rmSync(opts.sqliteFilename);
       } catch (error) {
-        console.log("could not delete", opts.sqliteFilename, error);
+        console.log("could not delete", opts.sqliteFilename);
+      }
+    }
+
+    // attempt to clean up the postgres db
+    if (opts.indexer === "postgres") {
+      try {
+        await cleanDatabase(drizzle(postgres(opts.databaseUrl)));
+      } catch (error) {
+        console.log("could not clean postgres database");
       }
     }
   }

diff --git a/e2e/pnpm-lock.yaml b/e2e/pnpm-lock.yaml
diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts
@@ -10,6 +10,7 @@ export * from "./hexToResource";
 export * from "./readHex";
 export * from "./resourceToHex";
 export * from "./resourceTypes";
+export * from "./result";
 export * from "./sendTransaction";
 export * from "./spliceHex";
 export * from "./transportObserver";

diff --git a/packages/common/src/result.ts b/packages/common/src/result.ts
@@ -0,0 +1,10 @@
+// Inspired by https://doc.rust-lang.org/std/result/
+export type Result<Ok, Err = unknown> = { ok: Ok } | { error: Err };
+
+export function isOk<Ok, Err>(result: Result<Ok, Err>): result is { ok: Ok } {
+  return "ok" in result;
+}
+
+export function isError<Ok, Err>(result: Result<Ok, Err>): result is { error: Err } {
+  return "error" in result;
+}
diff --git a/packages/common/src/utils/includes.ts b/packages/common/src/utils/includes.ts
@@ -0,0 +1,3 @@
+export function includes<item>(items: item[], value: any): value is item {
+  return items.includes(value);
+}
diff --git a/packages/common/src/utils/index.ts b/packages/common/src/utils/index.ts
@@ -6,6 +6,7 @@ export * from "./chunk";
 export * from "./curry";
 export * from "./groupBy";
 export * from "./identity";
+export * from "./includes";
 export * from "./isDefined";
 export * from "./isNotNull";
 export * from "./iteratorToArray";

diff --git a/packages/store-indexer/bin/postgres-frontend.ts b/packages/store-indexer/bin/postgres-frontend.ts
@@ -6,10 +6,11 @@ import cors from "@koa/cors";
 import Router from "@koa/router";
 import { createKoaMiddleware } from "trpc-koa-adapter";
 import { createAppRouter } from "@latticexyz/store-sync/trpc-indexer";
-import { createQueryAdapter } from "../src/postgres/createQueryAdapter";
 import { drizzle } from "drizzle-orm/postgres-js";
 import postgres from "postgres";
 import { frontendEnvSchema, parseEnv } from "./parseEnv";
+import { createQueryAdapter } from "../src/postgres/deprecated/createQueryAdapter";
+import { apiRoutes } from "../src/postgres/apiRoutes";
 
 const env = parseEnv(
   z.intersection(
@@ -20,10 +21,11 @@ const env = parseEnv(
   )
 );
 
-const database = drizzle(postgres(env.DATABASE_URL));
+const database = postgres(env.DATABASE_URL);
 
 const server = new Koa();
 server.use(cors());
+server.use(apiRoutes(database));
 
 const router = new Router();
 
@@ -47,7 +49,7 @@ server.use(
     prefix: "/trpc",
     router: createAppRouter(),
     createContext: async () => ({
-      queryAdapter: await createQueryAdapter(database),
+      queryAdapter: await createQueryAdapter(drizzle(database)),
     }),
   })
 );

diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts
@@ -39,7 +39,7 @@ const database = drizzle(postgres(env.DATABASE_URL));
 
 if (await shouldCleanDatabase(database, chainId)) {
   console.log("outdated database detected, clearing data to start fresh");
-  cleanDatabase(database);
+  await cleanDatabase(database);
 }
 
 const { storageAdapter, tables } = await createStorageAdapter({ database, publicClient });

diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts
@@ -16,6 +16,7 @@ import { createQueryAdapter } from "../src/sqlite/createQueryAdapter";
 import { isDefined } from "@latticexyz/common/utils";
 import { combineLatest, filter, first } from "rxjs";
 import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv";
+import { apiRoutes } from "../src/sqlite/apiRoutes";
 
 const env = parseEnv(
   z.intersection(
@@ -91,6 +92,7 @@ combineLatest([latestBlockNumber$, storedBlockLogs$])
 
 const server = new Koa();
 server.use(cors());
+server.use(apiRoutes(database));
 
 const router = new Router();
 

diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json
@@ -44,23 +44,27 @@
     "@latticexyz/store-sync": "workspace:*",
     "@trpc/client": "10.34.0",
     "@trpc/server": "10.34.0",
+    "accepts": "^1.3.8",
     "better-sqlite3": "^8.6.0",
     "debug": "^4.3.4",
     "dotenv": "^16.0.3",
     "drizzle-orm": "^0.28.5",
     "koa": "^2.14.2",
-    "postgres": "^3.3.5",
+    "koa-compose": "^4.1.0",
+    "postgres": "3.3.5",
     "rxjs": "7.5.5",
     "superjson": "^1.12.4",
     "trpc-koa-adapter": "^1.1.3",
     "viem": "1.14.0",
     "zod": "^3.21.4"
   },
   "devDependencies": {
+    "@types/accepts": "^1.3.7",
     "@types/better-sqlite3": "^7.6.4",
     "@types/cors": "^2.8.13",
     "@types/debug": "^4.1.7",
     "@types/koa": "^2.13.12",
+    "@types/koa-compose": "^3.2.8",
     "@types/koa__cors": "^4.0.3",
     "@types/koa__router": "^12.0.4",
     "concurrently": "^8.2.2",