From e23cc0057c713121230aef6d03e7b0944dc9192a Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 17:35:51 +0300 Subject: [PATCH 1/8] feat(store-indexer): separate postgres indexer/frontend services --- packages/store-indexer/bin/parseEnv.ts | 16 +++--- .../store-indexer/bin/postgres-frontend.ts | 49 +++++++++++++++++++ .../store-indexer/bin/postgres-indexer.ts | 36 ++++---------- packages/store-indexer/bin/sqlite-indexer.ts | 13 +++-- packages/store-indexer/package.json | 1 + packages/store-indexer/tsup.config.ts | 2 +- 6 files changed, 77 insertions(+), 40 deletions(-) create mode 100644 packages/store-indexer/bin/postgres-frontend.ts diff --git a/packages/store-indexer/bin/parseEnv.ts b/packages/store-indexer/bin/parseEnv.ts index 7e223c1a03..cc145a80fe 100644 --- a/packages/store-indexer/bin/parseEnv.ts +++ b/packages/store-indexer/bin/parseEnv.ts @@ -1,9 +1,12 @@ -import { z, ZodError, ZodIntersection, ZodTypeAny } from "zod"; +import { z, ZodError, ZodTypeAny } from "zod"; -const commonSchema = z.intersection( +export const frontendEnvSchema = z.object({ + HOST: z.string().default("0.0.0.0"), + PORT: z.coerce.number().positive().default(3001), +}); + +export const indexerEnvSchema = z.intersection( z.object({ - HOST: z.string().default("0.0.0.0"), - PORT: z.coerce.number().positive().default(3001), START_BLOCK: z.coerce.bigint().nonnegative().default(0n), MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n), POLLING_INTERVAL: z.coerce.number().positive().default(1000), @@ -20,10 +23,7 @@ const commonSchema = z.intersection( ]) ); -export function parseEnv( - schema?: TSchema -): z.infer : typeof commonSchema> { - const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema; +export function parseEnv(envSchema: TSchema): z.infer { try { return envSchema.parse(process.env); } catch (error) { diff --git a/packages/store-indexer/bin/postgres-frontend.ts b/packages/store-indexer/bin/postgres-frontend.ts new file mode 100644 index 0000000000..fdfcab402e --- /dev/null +++ b/packages/store-indexer/bin/postgres-frontend.ts @@ -0,0 +1,49 @@ +#!/usr/bin/env node +import "dotenv/config"; +import { z } from "zod"; +import { DefaultLogger } from "drizzle-orm"; +import fastify from "fastify"; +import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify"; +import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer"; +import { createQueryAdapter } from "../src/postgres/createQueryAdapter"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { frontendEnvSchema, parseEnv } from "./parseEnv"; + +const env = parseEnv( + z.intersection( + frontendEnvSchema, + z.object({ + DATABASE_URL: z.string(), + }) + ) +); + +const database = drizzle(postgres(env.DATABASE_URL), { + logger: new DefaultLogger(), +}); + +// @see https://fastify.dev/docs/latest/ +const server = fastify({ + maxParamLength: 5000, +}); + +await server.register(import("@fastify/cors")); + +// k8s healthchecks +server.get("/healthz", (req, res) => res.code(200).send()); +server.get("/readyz", (req, res) => res.code(200).send()); + +// @see https://trpc.io/docs/server/adapters/fastify +server.register(fastifyTRPCPlugin, { + prefix: "/trpc", + trpcOptions: { + router: createAppRouter(), + createContext: async () => ({ + queryAdapter: await createQueryAdapter(database), + }), + }, +}); + +await server.listen({ host: env.HOST, port: env.PORT }); +console.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`); diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 51d9a8c6b1..0bddcab64d 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -3,22 +3,22 @@ import "dotenv/config"; import { z } from "zod"; import { DefaultLogger, eq } from "drizzle-orm"; import { createPublicClient, fallback, webSocket, http, Transport } from "viem"; -import fastify from "fastify"; -import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify"; -import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer"; -import { createQueryAdapter } from "../src/postgres/createQueryAdapter"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; -import { parseEnv } from "./parseEnv"; +import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv"; +import fastify from "fastify"; const env = parseEnv( - z.object({ - DATABASE_URL: z.string(), - }) + z.intersection( + z.intersection(indexerEnvSchema, frontendEnvSchema), + z.object({ + DATABASE_URL: z.string(), + }) + ) ); const transports: Transport[] = [ @@ -93,27 +93,11 @@ combineLatest([latestBlockNumber$, storedBlockLogs$]) console.log("all caught up"); }); -// @see https://fastify.dev/docs/latest/ -const server = fastify({ - maxParamLength: 5000, -}); - -await server.register(import("@fastify/cors")); +const server = fastify(); // k8s healthchecks server.get("/healthz", (req, res) => res.code(200).send()); server.get("/readyz", (req, res) => (isCaughtUp ? res.code(200).send("ready") : res.code(424).send("backfilling"))); -// @see https://trpc.io/docs/server/adapters/fastify -server.register(fastifyTRPCPlugin, { - prefix: "/trpc", - trpcOptions: { - router: createAppRouter(), - createContext: async () => ({ - queryAdapter: await createQueryAdapter(database), - }), - }, -}); - await server.listen({ host: env.HOST, port: env.PORT }); -console.log(`indexer server listening on http://${env.HOST}:${env.PORT}`); +console.log(`postgres indexer healthcheck server listening on http://${env.HOST}:${env.PORT}`); diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts index b880c4b6d5..b664c9de38 100644 --- a/packages/store-indexer/bin/sqlite-indexer.ts +++ b/packages/store-indexer/bin/sqlite-indexer.ts @@ -13,12 +13,15 @@ import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/ import { createQueryAdapter } from "../src/sqlite/createQueryAdapter"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; -import { parseEnv } from "./parseEnv"; +import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv"; const env = parseEnv( - z.object({ - SQLITE_FILENAME: z.string().default("indexer.db"), - }) + z.intersection( + z.intersection(indexerEnvSchema, frontendEnvSchema), + z.object({ + SQLITE_FILENAME: z.string().default("indexer.db"), + }) + ) ); const transports: Transport[] = [ @@ -106,4 +109,4 @@ server.register(fastifyTRPCPlugin, { }); await server.listen({ host: env.HOST, port: env.PORT }); -console.log(`indexer server listening on http://${env.HOST}:${env.PORT}`); +console.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`); diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json index 674460dae3..759aa8e443 100644 --- a/packages/store-indexer/package.json +++ b/packages/store-indexer/package.json @@ -14,6 +14,7 @@ }, "types": "src/index.ts", "bin": { + "postgres-frontend": "./dist/bin/postgres-frontend.js", "postgres-indexer": "./dist/bin/postgres-indexer.js", "sqlite-indexer": "./dist/bin/sqlite-indexer.js" }, diff --git a/packages/store-indexer/tsup.config.ts b/packages/store-indexer/tsup.config.ts index ab77d021ff..97c8390fcb 100644 --- a/packages/store-indexer/tsup.config.ts +++ b/packages/store-indexer/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "bin/postgres-indexer.ts", "bin/sqlite-indexer.ts"], + entry: ["src/index.ts", "bin/postgres-frontend.ts", "bin/postgres-indexer.ts", "bin/sqlite-indexer.ts"], target: "esnext", format: ["esm"], dts: false, From b8e4c60d71c1ba065c5b7f21d630c9ca3d04c793 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 17:42:49 +0300 Subject: [PATCH 2/8] run both postgres services with concurrently --- packages/store-indexer/package.json | 3 +- pnpm-lock.yaml | 54 ++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json index 759aa8e443..169b205ec0 100644 --- a/packages/store-indexer/package.json +++ b/packages/store-indexer/package.json @@ -25,7 +25,7 @@ "clean:js": "rimraf dist", "dev": "tsup --watch", "lint": "eslint .", - "start:postgres": "tsx bin/postgres-indexer", + "start:postgres": "concurrently -n indexer,frontend -c cyan,magenta 'tsx bin/postgres-indexer' 'tsx bin/postgres-frontend'", "start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres", "start:postgres:testnet": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres", "start:sqlite": "tsx bin/sqlite-indexer", @@ -59,6 +59,7 @@ "@types/better-sqlite3": "^7.6.4", "@types/cors": "^2.8.13", "@types/debug": "^4.1.7", + "concurrently": "^8.2.2", "tsup": "^6.7.0", "tsx": "^3.12.6", "vitest": "0.31.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 400a0ab169..0df9596050 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -882,6 +882,9 @@ importers: '@types/debug': specifier: ^4.1.7 version: 4.1.7 + concurrently: + specifier: ^8.2.2 + version: 8.2.2 tsup: specifier: ^6.7.0 version: 6.7.0(postcss@8.4.23)(typescript@5.1.6) @@ -4691,6 +4694,22 @@ packages: well-known-symbols: 2.0.0 dev: true + /concurrently@8.2.2: + resolution: {integrity: sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==} + engines: {node: ^14.13.0 || >=16.0.0} + hasBin: true + dependencies: + chalk: 4.1.2 + date-fns: 2.30.0 + lodash: 4.17.21 + rxjs: 7.8.1 + shell-quote: 1.8.1 + spawn-command: 0.0.2 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + dev: true + /convert-source-map@1.9.0: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} dev: true @@ -4865,6 +4884,13 @@ packages: resolution: {integrity: sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==} dev: true + /date-fns@2.30.0: + resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} + engines: {node: '>=0.11'} + dependencies: + '@babel/runtime': 7.21.0 + dev: true + /date-time@3.1.0: resolution: {integrity: sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==} engines: {node: '>=6'} @@ -9671,6 +9697,12 @@ packages: tslib: 2.5.0 dev: false + /rxjs@7.8.1: + resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==} + dependencies: + tslib: 2.5.0 + dev: true + /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -9809,6 +9841,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + /shell-quote@1.8.1: + resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==} + dev: true + /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -9961,6 +9997,10 @@ packages: whatwg-url: 7.1.0 dev: true + /spawn-command@0.0.2: + resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==} + dev: true + /spawndamnit@2.0.0: resolution: {integrity: sha512-j4JKEcncSjFlqIwU5L/rp2N5SIPsdxaRsIv678+TZxZ0SRDJTm8JrxJMjE/XuiEZNEir3S8l0Fa3Ke339WI4qA==} dependencies: @@ -10569,7 +10609,6 @@ packages: /tslib@2.5.0: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} - dev: false /tsort@0.0.1: resolution: {integrity: sha512-Tyrf5mxF8Ofs1tNoxA13lFeZ2Zrbd6cKbuH3V+MQ5sb6DtBj5FjrXVsRWT8YvNAQTqNoz66dz1WsbigI22aEnw==} @@ -11428,6 +11467,19 @@ packages: y18n: 5.0.8 yargs-parser: 21.1.1 + /yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + dependencies: + cliui: 8.0.1 + escalade: 3.1.1 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + dev: true + /yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} From 333af6a0bcff8d435a3ae80e654e2794c316abc8 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 06:44:29 -0800 Subject: [PATCH 3/8] Create large-hounds-type.md --- .changeset/large-hounds-type.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/large-hounds-type.md diff --git a/.changeset/large-hounds-type.md b/.changeset/large-hounds-type.md new file mode 100644 index 0000000000..59c1d88773 --- /dev/null +++ b/.changeset/large-hounds-type.md @@ -0,0 +1,7 @@ +--- +"@latticexyz/store-indexer": major +--- + +Separated frontend server and indexer service for Postgres indexer. Now you can run the Postgres indexer with one writer and many readers. + +If you were previously using the `postgres-indexer` binary, you'll now need to run both `postgres-indexer` and `postgres-frontend`. From 042b7dca1505f394da6dab498d669ebd19fe732f Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 17:58:57 +0300 Subject: [PATCH 4/8] separate healthcheck host/port and run optionally --- .../store-indexer/bin/postgres-indexer.ts | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 0bddcab64d..75352f57cd 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -9,14 +9,15 @@ import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; -import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv"; -import fastify from "fastify"; +import { indexerEnvSchema, parseEnv } from "./parseEnv"; const env = parseEnv( z.intersection( - z.intersection(indexerEnvSchema, frontendEnvSchema), + indexerEnvSchema, z.object({ DATABASE_URL: z.string(), + HEALTHCHECK_HOST: z.string().optional(), + HEALTHCHECK_PORT: z.coerce.number().optional(), }) ) ); @@ -93,11 +94,16 @@ combineLatest([latestBlockNumber$, storedBlockLogs$]) console.log("all caught up"); }); -const server = fastify(); +if (env.HEALTHCHECK_HOST != null || env.HEALTHCHECK_PORT != null) { + const { default: fastify } = await import("fastify"); -// k8s healthchecks -server.get("/healthz", (req, res) => res.code(200).send()); -server.get("/readyz", (req, res) => (isCaughtUp ? res.code(200).send("ready") : res.code(424).send("backfilling"))); + const server = fastify(); -await server.listen({ host: env.HOST, port: env.PORT }); -console.log(`postgres indexer healthcheck server listening on http://${env.HOST}:${env.PORT}`); + // k8s healthchecks + server.get("/healthz", (req, res) => res.code(200).send()); + server.get("/readyz", (req, res) => (isCaughtUp ? res.code(200).send("ready") : res.code(424).send("backfilling"))); + + server.listen({ host: env.HEALTHCHECK_HOST, port: env.HEALTHCHECK_PORT }, (error, address) => { + console.log(`postgres indexer healthcheck server listening on ${address}`); + }); +} From 3e10db21797a7658acb5022ca2a2002556b087be Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 18:02:45 +0300 Subject: [PATCH 5/8] update readme --- packages/store-indexer/README.md | 37 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/store-indexer/README.md b/packages/store-indexer/README.md index 32630ec56a..26d33ee1d6 100644 --- a/packages/store-indexer/README.md +++ b/packages/store-indexer/README.md @@ -11,7 +11,7 @@ npm install @latticexyz/store-indexer npm sqlite-indexer # or -npm postgres-indexer +npm postgres-indexer & npm postgres-frontend ``` or execute the one of the package bins directly: @@ -19,32 +19,39 @@ or execute the one of the package bins directly: ```sh npx -p @latticexyz/store-indexer sqlite-indexer # or -npx -p @latticexyz/store-indexer postgres-indexer +npx -p @latticexyz/store-indexer postgres-indexer & npx -p @latticexyz/store-indexer postgres-frontend ``` ## Configuration Each indexer can be configured with environment variables. -### Common environment variables +### Common environment variables for indexer -| Variable | Description | Default | -| ------------------ | ---------------------------------------------------------- | --------- | -| `HOST` | Host that the indexer server listens on | `0.0.0.0` | -| `PORT` | Port that the indexer server listens on | `3001` | -| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | | -| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | | -| `START_BLOCK` | Block number to start indexing from | `0` | -| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` | -| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` | +| Variable | Description | Default | +| ------------------ | ---------------------------------------------------------- | ------- | +| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | | +| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | | +| `START_BLOCK` | Block number to start indexing from | `0` | +| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` | +| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` | Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues. +### Common environment variables for frontend + +| Variable | Description | Default | +| -------- | ------------------------------------------------ | --------- | +| `HOST` | Host that the indexer frontend server listens on | `0.0.0.0` | +| `PORT` | Port that the indexer frontend server listens on | `3001` | + ### Postgres indexer environment variables -| Variable | Description | Default | -| -------------- | ----------------------- | ------- | -| `DATABASE_URL` | Postgres connection URL | | +| Variable | Description | Default | +| ------------------ | --------------------------------------------------- | ------- | +| `DATABASE_URL` | Postgres connection URL | | +| `HEALTHCHECK_HOST` | Host that the indexer healthcheck server listens on | | +| `HEATHCHECK_PORT` | Port that the indexer healthcheck server listens on | | ### SQLite indexer environment variables From dc7ff8e5e4f786d6d10026c7de617dbf63714360 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 18:14:56 +0300 Subject: [PATCH 6/8] turn off logging --- packages/store-indexer/bin/postgres-indexer.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 75352f57cd..f853a9d8ee 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -35,9 +35,7 @@ const publicClient = createPublicClient({ }); const chainId = await publicClient.getChainId(); -const database = drizzle(postgres(env.DATABASE_URL), { - logger: new DefaultLogger(), -}); +const database = drizzle(postgres(env.DATABASE_URL)); const { storageAdapter, internalTables } = await postgresStorage({ database, publicClient }); From d2a40e08ba02a4d2d293008bd3c7d4041c82b03b Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 18:32:38 +0300 Subject: [PATCH 7/8] turn off all logging --- packages/store-indexer/bin/postgres-frontend.ts | 5 +---- packages/store-indexer/bin/postgres-indexer.ts | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/store-indexer/bin/postgres-frontend.ts b/packages/store-indexer/bin/postgres-frontend.ts index fdfcab402e..ced2dd7d5f 100644 --- a/packages/store-indexer/bin/postgres-frontend.ts +++ b/packages/store-indexer/bin/postgres-frontend.ts @@ -1,7 +1,6 @@ #!/usr/bin/env node import "dotenv/config"; import { z } from "zod"; -import { DefaultLogger } from "drizzle-orm"; import fastify from "fastify"; import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify"; import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer"; @@ -19,9 +18,7 @@ const env = parseEnv( ) ); -const database = drizzle(postgres(env.DATABASE_URL), { - logger: new DefaultLogger(), -}); +const database = drizzle(postgres(env.DATABASE_URL)); // @see https://fastify.dev/docs/latest/ const server = fastify({ diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index f853a9d8ee..025fe818b0 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import "dotenv/config"; import { z } from "zod"; -import { DefaultLogger, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { createPublicClient, fallback, webSocket, http, Transport } from "viem"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; From c63ee02141e55b55b4dbba40229a92f650c0ab26 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Thu, 9 Nov 2023 07:37:03 -0800 Subject: [PATCH 8/8] Update large-hounds-type.md --- .changeset/large-hounds-type.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/large-hounds-type.md b/.changeset/large-hounds-type.md index 59c1d88773..16996091bb 100644 --- a/.changeset/large-hounds-type.md +++ b/.changeset/large-hounds-type.md @@ -5,3 +5,5 @@ Separated frontend server and indexer service for Postgres indexer. Now you can run the Postgres indexer with one writer and many readers. If you were previously using the `postgres-indexer` binary, you'll now need to run both `postgres-indexer` and `postgres-frontend`. + +For consistency, the Postgres database logs are now disabled by default. If you were using these, please let us know so we can add them back in with an environment variable flag.