From 5ecccfe751b0d217f98a45e8e7fdc73d15ad6494 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 15:37:18 +0000 Subject: [PATCH] feat(store-indexer): separate postgres indexer/frontend services (#1887) --- .changeset/large-hounds-type.md | 9 ++++ packages/store-indexer/README.md | 37 +++++++------ packages/store-indexer/bin/parseEnv.ts | 16 +++--- .../store-indexer/bin/postgres-frontend.ts | 46 ++++++++++++++++ .../store-indexer/bin/postgres-indexer.ts | 54 ++++++++----------- packages/store-indexer/bin/sqlite-indexer.ts | 13 +++-- packages/store-indexer/package.json | 4 +- packages/store-indexer/tsup.config.ts | 2 +- pnpm-lock.yaml | 54 ++++++++++++++++++- 9 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 .changeset/large-hounds-type.md create mode 100644 packages/store-indexer/bin/postgres-frontend.ts diff --git a/.changeset/large-hounds-type.md b/.changeset/large-hounds-type.md new file mode 100644 index 0000000000..16996091bb --- /dev/null +++ b/.changeset/large-hounds-type.md @@ -0,0 +1,9 @@ +--- +"@latticexyz/store-indexer": major +--- + +Separated frontend server and indexer service for Postgres indexer. Now you can run the Postgres indexer with one writer and many readers. + +If you were previously using the `postgres-indexer` binary, you'll now need to run both `postgres-indexer` and `postgres-frontend`. + +For consistency, the Postgres database logs are now disabled by default. If you were using these, please let us know so we can add them back in with an environment variable flag. diff --git a/packages/store-indexer/README.md b/packages/store-indexer/README.md index 32630ec56a..26d33ee1d6 100644 --- a/packages/store-indexer/README.md +++ b/packages/store-indexer/README.md @@ -11,7 +11,7 @@ npm install @latticexyz/store-indexer npm sqlite-indexer # or -npm postgres-indexer +npm postgres-indexer & npm postgres-frontend ``` or execute the one of the package bins directly: @@ -19,32 +19,39 @@ or execute the one of the package bins directly: ```sh npx -p @latticexyz/store-indexer sqlite-indexer # or -npx -p @latticexyz/store-indexer postgres-indexer +npx -p @latticexyz/store-indexer postgres-indexer & npx -p @latticexyz/store-indexer postgres-frontend ``` ## Configuration Each indexer can be configured with environment variables. -### Common environment variables +### Common environment variables for indexer -| Variable | Description | Default | -| ------------------ | ---------------------------------------------------------- | --------- | -| `HOST` | Host that the indexer server listens on | `0.0.0.0` | -| `PORT` | Port that the indexer server listens on | `3001` | -| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | | -| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | | -| `START_BLOCK` | Block number to start indexing from | `0` | -| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` | -| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` | +| Variable | Description | Default | +| ------------------ | ---------------------------------------------------------- | ------- | +| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | | +| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | | +| `START_BLOCK` | Block number to start indexing from | `0` | +| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` | +| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` | Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues. +### Common environment variables for frontend + +| Variable | Description | Default | +| -------- | ------------------------------------------------ | --------- | +| `HOST` | Host that the indexer frontend server listens on | `0.0.0.0` | +| `PORT` | Port that the indexer frontend server listens on | `3001` | + ### Postgres indexer environment variables -| Variable | Description | Default | -| -------------- | ----------------------- | ------- | -| `DATABASE_URL` | Postgres connection URL | | +| Variable | Description | Default | +| ------------------ | --------------------------------------------------- | ------- | +| `DATABASE_URL` | Postgres connection URL | | +| `HEALTHCHECK_HOST` | Host that the indexer healthcheck server listens on | | +| `HEATHCHECK_PORT` | Port that the indexer healthcheck server listens on | | ### SQLite indexer environment variables diff --git a/packages/store-indexer/bin/parseEnv.ts b/packages/store-indexer/bin/parseEnv.ts index 7e223c1a03..cc145a80fe 100644 --- a/packages/store-indexer/bin/parseEnv.ts +++ b/packages/store-indexer/bin/parseEnv.ts @@ -1,9 +1,12 @@ -import { z, ZodError, ZodIntersection, ZodTypeAny } from "zod"; +import { z, ZodError, ZodTypeAny } from "zod"; -const commonSchema = z.intersection( +export const frontendEnvSchema = z.object({ + HOST: z.string().default("0.0.0.0"), + PORT: z.coerce.number().positive().default(3001), +}); + +export const indexerEnvSchema = z.intersection( z.object({ - HOST: z.string().default("0.0.0.0"), - PORT: z.coerce.number().positive().default(3001), START_BLOCK: z.coerce.bigint().nonnegative().default(0n), MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n), POLLING_INTERVAL: z.coerce.number().positive().default(1000), @@ -20,10 +23,7 @@ const commonSchema = z.intersection( ]) ); -export function parseEnv( - schema?: TSchema -): z.infer : typeof commonSchema> { - const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema; +export function parseEnv(envSchema: TSchema): z.infer { try { return envSchema.parse(process.env); } catch (error) { diff --git a/packages/store-indexer/bin/postgres-frontend.ts b/packages/store-indexer/bin/postgres-frontend.ts new file mode 100644 index 0000000000..ced2dd7d5f --- /dev/null +++ b/packages/store-indexer/bin/postgres-frontend.ts @@ -0,0 +1,46 @@ +#!/usr/bin/env node +import "dotenv/config"; +import { z } from "zod"; +import fastify from "fastify"; +import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify"; +import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer"; +import { createQueryAdapter } from "../src/postgres/createQueryAdapter"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { frontendEnvSchema, parseEnv } from "./parseEnv"; + +const env = parseEnv( + z.intersection( + frontendEnvSchema, + z.object({ + DATABASE_URL: z.string(), + }) + ) +); + +const database = drizzle(postgres(env.DATABASE_URL)); + +// @see https://fastify.dev/docs/latest/ +const server = fastify({ + maxParamLength: 5000, +}); + +await server.register(import("@fastify/cors")); + +// k8s healthchecks +server.get("/healthz", (req, res) => res.code(200).send()); +server.get("/readyz", (req, res) => res.code(200).send()); + +// @see https://trpc.io/docs/server/adapters/fastify +server.register(fastifyTRPCPlugin, { + prefix: "/trpc", + trpcOptions: { + router: createAppRouter(), + createContext: async () => ({ + queryAdapter: await createQueryAdapter(database), + }), + }, +}); + +await server.listen({ host: env.HOST, port: env.PORT }); +console.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`); diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 51d9a8c6b1..025fe818b0 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -1,24 +1,25 @@ #!/usr/bin/env node import "dotenv/config"; import { z } from "zod"; -import { DefaultLogger, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { createPublicClient, fallback, webSocket, http, Transport } from "viem"; -import fastify from "fastify"; -import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify"; -import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer"; -import { createQueryAdapter } from "../src/postgres/createQueryAdapter"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; -import { parseEnv } from "./parseEnv"; +import { indexerEnvSchema, parseEnv } from "./parseEnv"; const env = parseEnv( - z.object({ - DATABASE_URL: z.string(), - }) + z.intersection( + indexerEnvSchema, + z.object({ + DATABASE_URL: z.string(), + HEALTHCHECK_HOST: z.string().optional(), + HEALTHCHECK_PORT: z.coerce.number().optional(), + }) + ) ); const transports: Transport[] = [ @@ -34,9 +35,7 @@ const publicClient = createPublicClient({ }); const chainId = await publicClient.getChainId(); -const database = drizzle(postgres(env.DATABASE_URL), { - logger: new DefaultLogger(), -}); +const database = drizzle(postgres(env.DATABASE_URL)); const { storageAdapter, internalTables } = await postgresStorage({ database, publicClient }); @@ -93,27 +92,16 @@ combineLatest([latestBlockNumber$, storedBlockLogs$]) console.log("all caught up"); }); -// @see https://fastify.dev/docs/latest/ -const server = fastify({ - maxParamLength: 5000, -}); +if (env.HEALTHCHECK_HOST != null || env.HEALTHCHECK_PORT != null) { + const { default: fastify } = await import("fastify"); -await server.register(import("@fastify/cors")); + const server = fastify(); -// k8s healthchecks -server.get("/healthz", (req, res) => res.code(200).send()); -server.get("/readyz", (req, res) => (isCaughtUp ? res.code(200).send("ready") : res.code(424).send("backfilling"))); + // k8s healthchecks + server.get("/healthz", (req, res) => res.code(200).send()); + server.get("/readyz", (req, res) => (isCaughtUp ? res.code(200).send("ready") : res.code(424).send("backfilling"))); -// @see https://trpc.io/docs/server/adapters/fastify -server.register(fastifyTRPCPlugin, { - prefix: "/trpc", - trpcOptions: { - router: createAppRouter(), - createContext: async () => ({ - queryAdapter: await createQueryAdapter(database), - }), - }, -}); - -await server.listen({ host: env.HOST, port: env.PORT }); -console.log(`indexer server listening on http://${env.HOST}:${env.PORT}`); + server.listen({ host: env.HEALTHCHECK_HOST, port: env.HEALTHCHECK_PORT }, (error, address) => { + console.log(`postgres indexer healthcheck server listening on ${address}`); + }); +} diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts index b880c4b6d5..b664c9de38 100644 --- a/packages/store-indexer/bin/sqlite-indexer.ts +++ b/packages/store-indexer/bin/sqlite-indexer.ts @@ -13,12 +13,15 @@ import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/ import { createQueryAdapter } from "../src/sqlite/createQueryAdapter"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; -import { parseEnv } from "./parseEnv"; +import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv"; const env = parseEnv( - z.object({ - SQLITE_FILENAME: z.string().default("indexer.db"), - }) + z.intersection( + z.intersection(indexerEnvSchema, frontendEnvSchema), + z.object({ + SQLITE_FILENAME: z.string().default("indexer.db"), + }) + ) ); const transports: Transport[] = [ @@ -106,4 +109,4 @@ server.register(fastifyTRPCPlugin, { }); await server.listen({ host: env.HOST, port: env.PORT }); -console.log(`indexer server listening on http://${env.HOST}:${env.PORT}`); +console.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`); diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json index 674460dae3..169b205ec0 100644 --- a/packages/store-indexer/package.json +++ b/packages/store-indexer/package.json @@ -14,6 +14,7 @@ }, "types": "src/index.ts", "bin": { + "postgres-frontend": "./dist/bin/postgres-frontend.js", "postgres-indexer": "./dist/bin/postgres-indexer.js", "sqlite-indexer": "./dist/bin/sqlite-indexer.js" }, @@ -24,7 +25,7 @@ "clean:js": "rimraf dist", "dev": "tsup --watch", "lint": "eslint .", - "start:postgres": "tsx bin/postgres-indexer", + "start:postgres": "concurrently -n indexer,frontend -c cyan,magenta 'tsx bin/postgres-indexer' 'tsx bin/postgres-frontend'", "start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres", "start:postgres:testnet": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres", "start:sqlite": "tsx bin/sqlite-indexer", @@ -58,6 +59,7 @@ "@types/better-sqlite3": "^7.6.4", "@types/cors": "^2.8.13", "@types/debug": "^4.1.7", + "concurrently": "^8.2.2", "tsup": "^6.7.0", "tsx": "^3.12.6", "vitest": "0.31.4" diff --git a/packages/store-indexer/tsup.config.ts b/packages/store-indexer/tsup.config.ts index ab77d021ff..97c8390fcb 100644 --- a/packages/store-indexer/tsup.config.ts +++ b/packages/store-indexer/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "bin/postgres-indexer.ts", "bin/sqlite-indexer.ts"], + entry: ["src/index.ts", "bin/postgres-frontend.ts", "bin/postgres-indexer.ts", "bin/sqlite-indexer.ts"], target: "esnext", format: ["esm"], dts: false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 400a0ab169..0df9596050 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -882,6 +882,9 @@ importers: '@types/debug': specifier: ^4.1.7 version: 4.1.7 + concurrently: + specifier: ^8.2.2 + version: 8.2.2 tsup: specifier: ^6.7.0 version: 6.7.0(postcss@8.4.23)(typescript@5.1.6) @@ -4691,6 +4694,22 @@ packages: well-known-symbols: 2.0.0 dev: true + /concurrently@8.2.2: + resolution: {integrity: sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==} + engines: {node: ^14.13.0 || >=16.0.0} + hasBin: true + dependencies: + chalk: 4.1.2 + date-fns: 2.30.0 + lodash: 4.17.21 + rxjs: 7.8.1 + shell-quote: 1.8.1 + spawn-command: 0.0.2 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + dev: true + /convert-source-map@1.9.0: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} dev: true @@ -4865,6 +4884,13 @@ packages: resolution: {integrity: sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==} dev: true + /date-fns@2.30.0: + resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} + engines: {node: '>=0.11'} + dependencies: + '@babel/runtime': 7.21.0 + dev: true + /date-time@3.1.0: resolution: {integrity: sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==} engines: {node: '>=6'} @@ -9671,6 +9697,12 @@ packages: tslib: 2.5.0 dev: false + /rxjs@7.8.1: + resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==} + dependencies: + tslib: 2.5.0 + dev: true + /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -9809,6 +9841,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + /shell-quote@1.8.1: + resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==} + dev: true + /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -9961,6 +9997,10 @@ packages: whatwg-url: 7.1.0 dev: true + /spawn-command@0.0.2: + resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==} + dev: true + /spawndamnit@2.0.0: resolution: {integrity: sha512-j4JKEcncSjFlqIwU5L/rp2N5SIPsdxaRsIv678+TZxZ0SRDJTm8JrxJMjE/XuiEZNEir3S8l0Fa3Ke339WI4qA==} dependencies: @@ -10569,7 +10609,6 @@ packages: /tslib@2.5.0: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} - dev: false /tsort@0.0.1: resolution: {integrity: sha512-Tyrf5mxF8Ofs1tNoxA13lFeZ2Zrbd6cKbuH3V+MQ5sb6DtBj5FjrXVsRWT8YvNAQTqNoz66dz1WsbigI22aEnw==} @@ -11428,6 +11467,19 @@ packages: y18n: 5.0.8 yargs-parser: 21.1.1 + /yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + dependencies: + cliui: 8.0.1 + escalade: 3.1.1 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + dev: true + /yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'}