From b3c22a183c0b288b9eb1487e4fef125bf7dae915 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 18 Sep 2023 21:25:08 +0100 Subject: [PATCH] refactor(store-indexer): add readme, refactor common env (#1533) --- .changeset/thin-rice-trade.md | 5 ++ packages/store-indexer/README.md | 53 +++++++++++++++++++ packages/store-indexer/bin/parseEnv.ts | 30 +++++++++++ .../store-indexer/bin/postgres-indexer.ts | 29 +++------- packages/store-indexer/bin/sqlite-indexer.ts | 29 +++------- 5 files changed, 100 insertions(+), 46 deletions(-) create mode 100644 .changeset/thin-rice-trade.md create mode 100644 packages/store-indexer/README.md create mode 100644 packages/store-indexer/bin/parseEnv.ts diff --git a/.changeset/thin-rice-trade.md b/.changeset/thin-rice-trade.md new file mode 100644 index 0000000000..13d4781cbe --- /dev/null +++ b/.changeset/thin-rice-trade.md @@ -0,0 +1,5 @@ +--- +"@latticexyz/store-indexer": patch +--- + +Added README and refactored handling of common environment variables diff --git a/packages/store-indexer/README.md b/packages/store-indexer/README.md new file mode 100644 index 0000000000..32630ec56a --- /dev/null +++ b/packages/store-indexer/README.md @@ -0,0 +1,53 @@ +# store-indexer + +A minimal Typescript indexer for [MUD Store](https://mud.dev/store) events (built on [store-sync](https://npmjs.com/package/@latticexyz/store-sync)) + +## Usage + +Install and run with: + +```sh +npm install @latticexyz/store-indexer + +npm sqlite-indexer +# or +npm postgres-indexer +``` + +or execute the one of the package bins directly: + +```sh +npx -p @latticexyz/store-indexer sqlite-indexer +# or +npx -p @latticexyz/store-indexer postgres-indexer +``` + +## Configuration + +Each indexer can be configured with environment variables. + +### Common environment variables + +| Variable | Description | Default | +| ------------------ | ---------------------------------------------------------- | --------- | +| `HOST` | Host that the indexer server listens on | `0.0.0.0` | +| `PORT` | Port that the indexer server listens on | `3001` | +| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | | +| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | | +| `START_BLOCK` | Block number to start indexing from | `0` | +| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` | +| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` | + +Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues. + +### Postgres indexer environment variables + +| Variable | Description | Default | +| -------------- | ----------------------- | ------- | +| `DATABASE_URL` | Postgres connection URL | | + +### SQLite indexer environment variables + +| Variable | Description | Default | +| ----------------- | ------------------------ | ------------ | +| `SQLITE_FILENAME` | SQLite database filename | `indexer.db` | diff --git a/packages/store-indexer/bin/parseEnv.ts b/packages/store-indexer/bin/parseEnv.ts new file mode 100644 index 0000000000..b4c3803b23 --- /dev/null +++ b/packages/store-indexer/bin/parseEnv.ts @@ -0,0 +1,30 @@ +import { isDefined } from "@latticexyz/common/utils"; +import { z, ZodIntersection, ZodTypeAny } from "zod"; + +const commonSchema = z.intersection( + z.object({ + HOST: z.string().default("0.0.0.0"), + PORT: z.coerce.number().positive().default(3001), + START_BLOCK: z.coerce.bigint().nonnegative().default(0n), + MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n), + POLLING_INTERVAL: z.coerce.number().positive().default(1000), + }), + z + .object({ + RPC_HTTP_URL: z.string(), + RPC_WS_URL: z.string(), + }) + .partial() + .refine((values) => Object.values(values).some(isDefined)) +); + +export function parseEnv( + schema?: TSchema +): z.infer : typeof commonSchema> { + const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema; + return envSchema.parse(process.env, { + errorMap: (issue) => ({ + message: `Missing or invalid environment variable: ${issue.path.join(".")}`, + }), + }); +} diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index b2b7068276..3a0428684a 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -13,30 +13,13 @@ import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; +import { parseEnv } from "./parseEnv"; -const env = z - .intersection( - z.object({ - HOST: z.string().default("0.0.0.0"), - PORT: z.coerce.number().positive().default(3001), - DATABASE_URL: z.string(), - START_BLOCK: z.coerce.bigint().nonnegative().default(0n), - MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n), - POLLING_INTERVAL: z.coerce.number().positive().default(1000), - }), - z - .object({ - RPC_HTTP_URL: z.string(), - RPC_WS_URL: z.string(), - }) - .partial() - .refine((values) => Object.values(values).some(isDefined)) - ) - .parse(process.env, { - errorMap: (issue) => ({ - message: `Missing or invalid environment variable: ${issue.path.join(".")}`, - }), - }); +const env = parseEnv( + z.object({ + DATABASE_URL: z.string(), + }) +); const transports: Transport[] = [ // prefer WS when specified diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts index 483dc9a823..afa4ab97bb 100644 --- a/packages/store-indexer/bin/sqlite-indexer.ts +++ b/packages/store-indexer/bin/sqlite-indexer.ts @@ -13,30 +13,13 @@ import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/ import { createQueryAdapter } from "../src/sqlite/createQueryAdapter"; import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; +import { parseEnv } from "./parseEnv"; -const env = z - .intersection( - z.object({ - HOST: z.string().default("0.0.0.0"), - PORT: z.coerce.number().positive().default(3001), - SQLITE_FILENAME: z.string().default("indexer.db"), - START_BLOCK: z.coerce.bigint().nonnegative().default(0n), - MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n), - POLLING_INTERVAL: z.coerce.number().positive().default(1000), - }), - z - .object({ - RPC_HTTP_URL: z.string(), - RPC_WS_URL: z.string(), - }) - .partial() - .refine((values) => Object.values(values).some(isDefined)) - ) - .parse(process.env, { - errorMap: (issue) => ({ - message: `Missing or invalid environment variable: ${issue.path.join(".")}`, - }), - }); +const env = parseEnv( + z.object({ + SQLITE_FILENAME: z.string().default("indexer.db"), + }) +); const transports: Transport[] = [ // prefer WS when specified