From e48fb3b037d2ee888a8c61a6fc51721c903559e3 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Fri, 1 Dec 2023 13:51:55 +0000 Subject: [PATCH] feat(store-indexer): clean database if outdated (#1984) --- .changeset/fair-bulldogs-decide.md | 5 ++ .changeset/tasty-cows-pay.md | 5 ++ .../store-indexer/bin/postgres-indexer.ts | 11 +++-- .../store-indexer/src/postgres/getLogs.ts | 4 +- .../src/postgres-decoded/cleanDatabase.ts | 27 ----------- .../createStorageAdapter.test.ts | 3 +- .../store-sync/src/postgres-decoded/index.ts | 1 - .../store-sync/src/postgres/cleanDatabase.ts | 48 +++++++++++++++---- .../src/postgres/createStorageAdapter.test.ts | 3 +- .../src/postgres/createStorageAdapter.ts | 6 ++- packages/store-sync/src/postgres/index.ts | 1 + .../src/postgres/setupTables.test.ts | 4 +- .../store-sync/src/postgres/setupTables.ts | 4 +- .../src/postgres/shouldCleanDatabase.ts | 36 ++++++++++++++ packages/store-sync/src/postgres/tables.ts | 7 +-- packages/store-sync/src/postgres/version.ts | 1 + 16 files changed, 113 insertions(+), 53 deletions(-) create mode 100644 .changeset/fair-bulldogs-decide.md create mode 100644 .changeset/tasty-cows-pay.md delete mode 100644 packages/store-sync/src/postgres-decoded/cleanDatabase.ts create mode 100644 packages/store-sync/src/postgres/shouldCleanDatabase.ts create mode 100644 packages/store-sync/src/postgres/version.ts diff --git a/.changeset/fair-bulldogs-decide.md b/.changeset/fair-bulldogs-decide.md new file mode 100644 index 0000000000..33fe835910 --- /dev/null +++ b/.changeset/fair-bulldogs-decide.md @@ -0,0 +1,5 @@ +--- +"@latticexyz/store-sync": major +--- + +Renamed singleton `chain` table to `config` table for clarity. diff --git a/.changeset/tasty-cows-pay.md b/.changeset/tasty-cows-pay.md new file mode 100644 index 0000000000..4099aaee5a --- /dev/null +++ b/.changeset/tasty-cows-pay.md @@ -0,0 +1,5 @@ +--- +"@latticexyz/store-indexer": minor +--- + +When the Postgres indexer starts up, it will now attempt to detect if the database is outdated and, if so, cleans up all MUD-related schemas and tables before proceeding. diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index eac1c1a09b..60903179b1 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -7,7 +7,7 @@ import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; -import { createStorageAdapter } from "@latticexyz/store-sync/postgres"; +import { cleanDatabase, createStorageAdapter, shouldCleanDatabase } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; import { indexerEnvSchema, parseEnv } from "./parseEnv"; @@ -37,6 +37,11 @@ const publicClient = createPublicClient({ const chainId = await publicClient.getChainId(); const database = drizzle(postgres(env.DATABASE_URL)); +if (await shouldCleanDatabase(database, chainId)) { + console.log("outdated database detected, clearing data to start fresh"); + cleanDatabase(database); +} + const { storageAdapter, tables } = await createStorageAdapter({ database, publicClient }); let startBlock = env.START_BLOCK; @@ -46,8 +51,8 @@ let startBlock = env.START_BLOCK; try { const chainState = await database .select() - .from(tables.chainTable) - .where(eq(tables.chainTable.chainId, chainId)) + .from(tables.configTable) + .where(eq(tables.configTable.chainId, chainId)) .limit(1) .execute() // Get the first record in a way that returns a possible `undefined` diff --git a/packages/store-indexer/src/postgres/getLogs.ts b/packages/store-indexer/src/postgres/getLogs.ts index 66860aee7c..b6e19aff14 100644 --- a/packages/store-indexer/src/postgres/getLogs.ts +++ b/packages/store-indexer/src/postgres/getLogs.ts @@ -41,8 +41,8 @@ export async function getLogs( // TODO: move the block number query into the records query for atomicity so we don't have to merge them here const chainState = await database .select() - .from(tables.chainTable) - .where(eq(tables.chainTable.chainId, chainId)) + .from(tables.configTable) + .where(eq(tables.configTable.chainId, chainId)) .limit(1) .execute() // Get the first record in a way that returns a possible `undefined` diff --git a/packages/store-sync/src/postgres-decoded/cleanDatabase.ts b/packages/store-sync/src/postgres-decoded/cleanDatabase.ts deleted file mode 100644 index 3e4773c866..0000000000 --- a/packages/store-sync/src/postgres-decoded/cleanDatabase.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; -import { getTables } from "./getTables"; -import { buildTable } from "./buildTable"; -import { isDefined, unique } from "@latticexyz/common/utils"; -import { debug } from "./debug"; -import { sql } from "drizzle-orm"; -import { pgDialect } from "../postgres/pgDialect"; -import { cleanDatabase as cleanBytesDatabase } from "../postgres/cleanDatabase"; - -// This intentionally just cleans up known schemas/tables. We could drop the database but that's scary. - -export async function cleanDatabase(db: PgDatabase): Promise { - const sqlTables = (await getTables(db)).map(buildTable); - - const schemaNames = unique(sqlTables.map((sqlTable) => getTableConfig(sqlTable).schema).filter(isDefined)); - - for (const schemaName of schemaNames) { - try { - debug(`dropping namespace ${schemaName} and all of its tables`); - await db.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); - } catch (error) { - debug(`failed to drop namespace ${schemaName}`, error); - } - } - - await cleanBytesDatabase(db); -} diff --git a/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts index db2427f0db..165a5f34c6 100644 --- a/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts @@ -47,11 +47,12 @@ describe("createStorageAdapter", async () => { await storageAdapter.storageAdapter(block); } - expect(await db.select().from(storageAdapter.tables.chainTable)).toMatchInlineSnapshot(` + expect(await db.select().from(storageAdapter.tables.configTable)).toMatchInlineSnapshot(` [ { "chainId": 31337, "lastUpdatedBlockNumber": 12n, + "version": "0.0.1", }, ] `); diff --git a/packages/store-sync/src/postgres-decoded/index.ts b/packages/store-sync/src/postgres-decoded/index.ts index 76ed5fcfb7..b2a569a50b 100644 --- a/packages/store-sync/src/postgres-decoded/index.ts +++ b/packages/store-sync/src/postgres-decoded/index.ts @@ -1,5 +1,4 @@ export * from "./buildTable"; -export * from "./cleanDatabase"; export * from "./createStorageAdapter"; export * from "./getTables"; export * from "./syncToPostgres"; diff --git a/packages/store-sync/src/postgres/cleanDatabase.ts b/packages/store-sync/src/postgres/cleanDatabase.ts index e63e897b28..b39a846e99 100644 --- a/packages/store-sync/src/postgres/cleanDatabase.ts +++ b/packages/store-sync/src/postgres/cleanDatabase.ts @@ -1,18 +1,46 @@ -import { PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; -import { tables } from "./tables"; -import { debug } from "./debug"; +import { PgDatabase, pgSchema, varchar } from "drizzle-orm/pg-core"; +import { debug as parentDebug } from "./debug"; import { sql } from "drizzle-orm"; import { pgDialect } from "./pgDialect"; -import { isDefined, unique } from "@latticexyz/common/utils"; +import { isNotNull } from "@latticexyz/common/utils"; -// This intentionally just cleans up known schemas/tables/rows. We could drop the database but that's scary. +const debug = parentDebug.extend("cleanDatabase"); +// Postgres internal table +const schemata = pgSchema("information_schema").table("schemata", { + schemaName: varchar("schema_name", { length: 64 }), +}); + +function isMudSchemaName(schemaName: string): boolean { + // address-prefixed schemas like {address}__{namespace} used by decoded postgres tables + // optional prefix for schemas created in tests + if (/(^|__)0x[0-9a-f]{40}__/i.test(schemaName)) { + return true; + } + // schema for internal tables + // optional prefix for schemas created in tests + if (/(^|__)mud$/.test(schemaName)) { + return true; + } + // old schema for internal tables + // TODO: remove after a while + if (/__mud_internal$/.test(schemaName)) { + return true; + } + return false; +} + +/** + * VERY DESTRUCTIVE! Finds and drops all MUD indexer related schemas and tables. + * @internal + */ export async function cleanDatabase(db: PgDatabase): Promise { - const schemaNames = unique( - Object.values(tables) - .map((table) => getTableConfig(table).schema) - .filter(isDefined) - ); + const schemaNames = (await db.select({ schemaName: schemata.schemaName }).from(schemata).execute()) + .map((row) => row.schemaName) + .filter(isNotNull) + .filter(isMudSchemaName); + + debug(`dropping ${schemaNames.length} schemas`); await db.transaction(async (tx) => { for (const schemaName of schemaNames) { diff --git a/packages/store-sync/src/postgres/createStorageAdapter.test.ts b/packages/store-sync/src/postgres/createStorageAdapter.test.ts index 554c1315ee..a5d9bd6795 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.test.ts @@ -45,11 +45,12 @@ describe("createStorageAdapter", async () => { await storageAdapter.storageAdapter(block); } - expect(await db.select().from(storageAdapter.tables.chainTable)).toMatchInlineSnapshot(` + expect(await db.select().from(storageAdapter.tables.configTable)).toMatchInlineSnapshot(` [ { "chainId": 31337, "lastUpdatedBlockNumber": 12n, + "version": "0.0.1", }, ] `); diff --git a/packages/store-sync/src/postgres/createStorageAdapter.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts index da587559d4..c2a9c9a675 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -7,6 +7,7 @@ import { tables } from "./tables"; import { spliceHex } from "@latticexyz/common"; import { setupTables } from "./setupTables"; import { StorageAdapter, StorageAdapterBlock } from "../common"; +import { version } from "./version"; // Currently assumes one DB per chain ID @@ -195,13 +196,14 @@ export async function createStorageAdapter { describe("before running", () => { it("should be missing schemas", async () => { - await expect(db.select().from(tables.chainTable)).rejects.toThrow(/relation "\w+mud.chain" does not exist/); + await expect(db.select().from(tables.configTable)).rejects.toThrow(/relation "\w+mud.config" does not exist/); await expect(db.select().from(tables.recordsTable)).rejects.toThrow(/relation "\w+mud.records" does not exist/); }); }); @@ -29,7 +29,7 @@ describe("setupTables", async () => { }); it("should have schemas", async () => { - expect(await db.select().from(tables.chainTable)).toMatchInlineSnapshot("[]"); + expect(await db.select().from(tables.configTable)).toMatchInlineSnapshot("[]"); expect(await db.select().from(tables.recordsTable)).toMatchInlineSnapshot("[]"); }); }); diff --git a/packages/store-sync/src/postgres/setupTables.ts b/packages/store-sync/src/postgres/setupTables.ts index 94a2ae3762..70f404bb50 100644 --- a/packages/store-sync/src/postgres/setupTables.ts +++ b/packages/store-sync/src/postgres/setupTables.ts @@ -2,9 +2,11 @@ import { AnyPgColumn, PgTableWithColumns, PgDatabase, getTableConfig } from "dri import { getTableColumns, sql } from "drizzle-orm"; import { ColumnDataType } from "kysely"; import { isDefined, unique } from "@latticexyz/common/utils"; -import { debug } from "./debug"; +import { debug as parentDebug } from "./debug"; import { pgDialect } from "./pgDialect"; +const debug = parentDebug.extend("setupTables"); + export async function setupTables( db: PgDatabase, tables: PgTableWithColumns[] diff --git a/packages/store-sync/src/postgres/shouldCleanDatabase.ts b/packages/store-sync/src/postgres/shouldCleanDatabase.ts new file mode 100644 index 0000000000..5e96d11d0c --- /dev/null +++ b/packages/store-sync/src/postgres/shouldCleanDatabase.ts @@ -0,0 +1,36 @@ +import { PgDatabase } from "drizzle-orm/pg-core"; +import { tables } from "./tables"; +import { debug as parentDebug } from "./debug"; +import { version as expectedVersion } from "./version"; + +const debug = parentDebug.extend("shouldCleanDatabase"); + +/** + * @internal + */ +export async function shouldCleanDatabase(db: PgDatabase, expectedChainId: number): Promise { + try { + const config = (await db.select().from(tables.configTable).limit(1).execute()).find(() => true); + + if (!config) { + debug("no record found in config table"); + return true; + } + + if (config.version !== expectedVersion) { + debug(`configured version (${config.version}) did not match expected version (${expectedVersion})`); + return true; + } + + if (config.chainId !== expectedChainId) { + debug(`configured chain ID (${config.chainId}) did not match expected chain ID (${expectedChainId})`); + return true; + } + + return false; + } catch (error) { + console.error(error); + debug("error while querying config table"); + return true; + } +} diff --git a/packages/store-sync/src/postgres/tables.ts b/packages/store-sync/src/postgres/tables.ts index 11c5c85506..770c8297f4 100644 --- a/packages/store-sync/src/postgres/tables.ts +++ b/packages/store-sync/src/postgres/tables.ts @@ -1,4 +1,4 @@ -import { boolean, index, pgSchema, primaryKey } from "drizzle-orm/pg-core"; +import { boolean, index, pgSchema, primaryKey, varchar } from "drizzle-orm/pg-core"; import { transformSchemaName } from "./transformSchemaName"; import { asAddress, asBigInt, asHex, asNumber } from "./columnTypes"; @@ -7,7 +7,8 @@ const schemaName = transformSchemaName("mud"); /** * Singleton table for the state of the chain we're indexing */ -const chainTable = pgSchema(schemaName).table("chain", { +const configTable = pgSchema(schemaName).table("config", { + version: varchar("version").notNull(), chainId: asNumber("chain_id", "bigint").notNull().primaryKey(), lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric").notNull(), }); @@ -39,6 +40,6 @@ const recordsTable = pgSchema(schemaName).table( ); export const tables = { - chainTable, + configTable, recordsTable, }; diff --git a/packages/store-sync/src/postgres/version.ts b/packages/store-sync/src/postgres/version.ts new file mode 100644 index 0000000000..dbeaefe350 --- /dev/null +++ b/packages/store-sync/src/postgres/version.ts @@ -0,0 +1 @@ +export const version = "0.0.1";