feat(store-sync,store-indexer): add postgres support (#1338)

.github/workflows/e2e.yml

@@ -12,18 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: ghcr.io/latticexyz/postgres-wal-logical:latest
-        env:
-          POSTGRES_USER: runner
-          POSTGRES_DB: mode_ephemeral
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+        image: postgres:12.1-alpine
         ports:
           - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -56,5 +49,7 @@ jobs:
         uses: ./.github/actions/require-empty-diff
 
       - name: Run sync tests
+        env:
+          DATABASE_URL: "postgres://postgres@localhost:5432/postgres"
         working-directory: ./e2e/packages/sync-test
         run: pnpm test

.github/workflows/test.yml

@@ -10,6 +10,13 @@ jobs:
   test:
     name: Run tests
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:12.1-alpine
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -21,6 +28,8 @@ jobs:
         uses: ./.github/actions/build
 
       - name: Run tests
+        env:
+          DATABASE_URL: "postgres://postgres@localhost:5432/postgres"
         run: pnpm test
 
       - name: Generate gas reports

e2e/packages/sync-test/indexerSync.test.ts

@@ -23,6 +23,17 @@ import {
 import { range } from "@latticexyz/utils";
 import path from "node:path";
 import { rpcHttpUrl } from "./setup/constants";
+import { z } from "zod";
+
+const env = z
+  .object({
+    DATABASE_URL: z.string().default("postgres://127.0.0.1/postgres"),
+  })
+  .parse(process.env, {
+    errorMap: (issue) => ({
+      message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
+    }),
+  });
 
 describe("Sync from indexer", async () => {
   const asyncErrorHandler = createAsyncErrorHandler();
@@ -54,14 +65,21 @@ describe("Sync from indexer", async () => {
     expect(asyncErrorHandler.getErrors()[0]).toContain("error fetching initial state from indexer");
   });
 
-  describe("indexer online", () => {
+  describe.each([["sqlite"], ["postgres"]] as const)("%s indexer", (indexerType) => {
     let indexerIteration = 1;
     let indexer: ReturnType<typeof startIndexer>;
 
     beforeEach(async () => {
       // Start indexer
       const port = 3000 + indexerIteration++;
-      indexer = startIndexer(port, path.join(__dirname, `anvil-${port}.db`), rpcHttpUrl, asyncErrorHandler.reportError);
+      indexer = startIndexer({
+        port,
+        rpcHttpUrl,
+        reportError: asyncErrorHandler.reportError,
+        ...(indexerType === "postgres"
+          ? { indexer: "postgres", databaseUrl: env.DATABASE_URL }
+          : { indexer: "sqlite", sqliteFilename: path.join(__dirname, `anvil-${port}.db`) }),
+      });
       await indexer.doneSyncing;
     });
 

e2e/packages/sync-test/package.json

@@ -25,6 +25,7 @@
     "typescript": "5.1.6",
     "viem": "1.6.0",
     "vite": "^4.2.1",
-    "vitest": "^0.31.0"
+    "vitest": "^0.31.0",
+    "zod": "^3.22.2"
   }
 }

e2e/packages/sync-test/setup/startIndexer.ts

@@ -3,30 +3,43 @@ import { execa } from "execa";
 import { rmSync } from "node:fs";
 import path from "node:path";
 
-export function startIndexer(
-  port: number,
-  sqliteFilename: string,
-  rpcUrl: string,
-  reportError: (error: string) => void
-) {
+type IndexerOptions =
+  | {
+      indexer: "sqlite";
+      sqliteFilename: string;
+    }
+  | {
+      indexer: "postgres";
+      databaseUrl: string;
+    };
+
+type StartIndexerOptions = {
+  port: number;
+  rpcHttpUrl: string;
+  reportError: (error: string) => void;
+} & IndexerOptions;
+
+export function startIndexer(opts: StartIndexerOptions) {
   let resolve: () => void;
   let reject: (reason?: string) => void;
   const doneSyncing = new Promise<void>((res, rej) => {
     resolve = res;
     reject = rej;
   });
 
-  console.log(chalk.magenta("[indexer]:"), "start syncing");
+  const env = {
+    DEBUG: "mud:*",
+    PORT: opts.port.toString(),
+    CHAIN_ID: "31337",
+    RPC_HTTP_URL: opts.rpcHttpUrl,
+    SQLITE_FILENAME: opts.indexer === "sqlite" ? opts.sqliteFilename : undefined,
+    DATABASE_URL: opts.indexer === "postgres" ? opts.databaseUrl : undefined,
+  };
+  console.log(chalk.magenta("[indexer]:"), "starting indexer", env);
 
-  const proc = execa("pnpm", ["start"], {
+  const proc = execa("pnpm", opts.indexer === "postgres" ? ["start:postgres"] : ["start:sqlite"], {
     cwd: path.join(__dirname, "..", "..", "..", "..", "packages", "store-indexer"),
-    env: {
-      DEBUG: "mud:*",
-      PORT: port.toString(),
-      CHAIN_ID: "31337",
-      RPC_HTTP_URL: rpcUrl,
-      SQLITE_FILENAME: sqliteFilename,
-    },
+    env,
   });
 
   proc.on("error", (error) => {
@@ -56,10 +69,12 @@ export function startIndexer(
 
   function cleanUp() {
     // attempt to clean up sqlite file
-    try {
-      rmSync(sqliteFilename);
-    } catch (error) {
-      console.log("could not delete", sqliteFilename, error);
+    if (opts.indexer === "sqlite") {
+      try {
+        rmSync(opts.sqliteFilename);
+      } catch (error) {
+        console.log("could not delete", opts.sqliteFilename, error);
+      }
     }
   }
 
@@ -70,7 +85,7 @@ export function startIndexer(
   });
 
   return {
-    url: `http://127.0.0.1:${port}/trpc`,
+    url: `http://127.0.0.1:${opts.port}/trpc`,
     doneSyncing,
     process: proc,
     kill: () =>

packages/common/src/utils/identity.ts

@@ -0,0 +1,3 @@
+export function identity<T>(value: T): T {
+  return value;
+}

packages/common/src/utils/index.ts

@@ -4,6 +4,7 @@ export * from "./bigIntMin";
 export * from "./bigIntSort";
 export * from "./chunk";
 export * from "./curry";
+export * from "./identity";
 export * from "./isDefined";
 export * from "./isNotNull";
 export * from "./wait";