From e61cdaa1a81576bb4f001506dd9bd0a5cbaaa5c3 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 10:12:38 +0000 Subject: [PATCH 01/23] rename current postgres approach to postgres-decoded --- .../store-sync/src/{postgres => postgres-decoded}/buildColumn.ts | 0 .../src/{postgres => postgres-decoded}/buildInternalTables.ts | 0 .../src/{postgres => postgres-decoded}/buildTable.test.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/buildTable.ts | 0 .../src/{postgres => postgres-decoded}/cleanDatabase.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/columnTypes.ts | 0 packages/store-sync/src/{postgres => postgres-decoded}/debug.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/getTableKey.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/getTables.ts | 0 packages/store-sync/src/{postgres => postgres-decoded}/index.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/pgDialect.ts | 0 .../src/{postgres => postgres-decoded}/postgresStorage.test.ts | 0 .../src/{postgres => postgres-decoded}/postgresStorage.ts | 0 .../src/{postgres => postgres-decoded}/schemaVersion.ts | 0 .../src/{postgres => postgres-decoded}/setupTables.test.ts | 0 .../store-sync/src/{postgres => postgres-decoded}/setupTables.ts | 0 .../src/{postgres => postgres-decoded}/syncToPostgres.ts | 0 .../src/{postgres => postgres-decoded}/transformSchemaName.ts | 0 18 files changed, 0 insertions(+), 0 deletions(-) rename packages/store-sync/src/{postgres => postgres-decoded}/buildColumn.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/buildInternalTables.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/buildTable.test.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/buildTable.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/cleanDatabase.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/columnTypes.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/debug.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/getTableKey.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/getTables.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/index.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/pgDialect.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/postgresStorage.test.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/postgresStorage.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/schemaVersion.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/setupTables.test.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/setupTables.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/syncToPostgres.ts (100%) rename packages/store-sync/src/{postgres => postgres-decoded}/transformSchemaName.ts (100%) diff --git a/packages/store-sync/src/postgres/buildColumn.ts b/packages/store-sync/src/postgres-decoded/buildColumn.ts similarity index 100% rename from packages/store-sync/src/postgres/buildColumn.ts rename to packages/store-sync/src/postgres-decoded/buildColumn.ts diff --git a/packages/store-sync/src/postgres/buildInternalTables.ts b/packages/store-sync/src/postgres-decoded/buildInternalTables.ts similarity index 100% rename from packages/store-sync/src/postgres/buildInternalTables.ts rename to packages/store-sync/src/postgres-decoded/buildInternalTables.ts diff --git a/packages/store-sync/src/postgres/buildTable.test.ts b/packages/store-sync/src/postgres-decoded/buildTable.test.ts similarity index 100% rename from packages/store-sync/src/postgres/buildTable.test.ts rename to packages/store-sync/src/postgres-decoded/buildTable.test.ts diff --git a/packages/store-sync/src/postgres/buildTable.ts b/packages/store-sync/src/postgres-decoded/buildTable.ts similarity index 100% rename from packages/store-sync/src/postgres/buildTable.ts rename to packages/store-sync/src/postgres-decoded/buildTable.ts diff --git a/packages/store-sync/src/postgres/cleanDatabase.ts b/packages/store-sync/src/postgres-decoded/cleanDatabase.ts similarity index 100% rename from packages/store-sync/src/postgres/cleanDatabase.ts rename to packages/store-sync/src/postgres-decoded/cleanDatabase.ts diff --git a/packages/store-sync/src/postgres/columnTypes.ts b/packages/store-sync/src/postgres-decoded/columnTypes.ts similarity index 100% rename from packages/store-sync/src/postgres/columnTypes.ts rename to packages/store-sync/src/postgres-decoded/columnTypes.ts diff --git a/packages/store-sync/src/postgres/debug.ts b/packages/store-sync/src/postgres-decoded/debug.ts similarity index 100% rename from packages/store-sync/src/postgres/debug.ts rename to packages/store-sync/src/postgres-decoded/debug.ts diff --git a/packages/store-sync/src/postgres/getTableKey.ts b/packages/store-sync/src/postgres-decoded/getTableKey.ts similarity index 100% rename from packages/store-sync/src/postgres/getTableKey.ts rename to packages/store-sync/src/postgres-decoded/getTableKey.ts diff --git a/packages/store-sync/src/postgres/getTables.ts b/packages/store-sync/src/postgres-decoded/getTables.ts similarity index 100% rename from packages/store-sync/src/postgres/getTables.ts rename to packages/store-sync/src/postgres-decoded/getTables.ts diff --git a/packages/store-sync/src/postgres/index.ts b/packages/store-sync/src/postgres-decoded/index.ts similarity index 100% rename from packages/store-sync/src/postgres/index.ts rename to packages/store-sync/src/postgres-decoded/index.ts diff --git a/packages/store-sync/src/postgres/pgDialect.ts b/packages/store-sync/src/postgres-decoded/pgDialect.ts similarity index 100% rename from packages/store-sync/src/postgres/pgDialect.ts rename to packages/store-sync/src/postgres-decoded/pgDialect.ts diff --git a/packages/store-sync/src/postgres/postgresStorage.test.ts b/packages/store-sync/src/postgres-decoded/postgresStorage.test.ts similarity index 100% rename from packages/store-sync/src/postgres/postgresStorage.test.ts rename to packages/store-sync/src/postgres-decoded/postgresStorage.test.ts diff --git a/packages/store-sync/src/postgres/postgresStorage.ts b/packages/store-sync/src/postgres-decoded/postgresStorage.ts similarity index 100% rename from packages/store-sync/src/postgres/postgresStorage.ts rename to packages/store-sync/src/postgres-decoded/postgresStorage.ts diff --git a/packages/store-sync/src/postgres/schemaVersion.ts b/packages/store-sync/src/postgres-decoded/schemaVersion.ts similarity index 100% rename from packages/store-sync/src/postgres/schemaVersion.ts rename to packages/store-sync/src/postgres-decoded/schemaVersion.ts diff --git a/packages/store-sync/src/postgres/setupTables.test.ts b/packages/store-sync/src/postgres-decoded/setupTables.test.ts similarity index 100% rename from packages/store-sync/src/postgres/setupTables.test.ts rename to packages/store-sync/src/postgres-decoded/setupTables.test.ts diff --git a/packages/store-sync/src/postgres/setupTables.ts b/packages/store-sync/src/postgres-decoded/setupTables.ts similarity index 100% rename from packages/store-sync/src/postgres/setupTables.ts rename to packages/store-sync/src/postgres-decoded/setupTables.ts diff --git a/packages/store-sync/src/postgres/syncToPostgres.ts b/packages/store-sync/src/postgres-decoded/syncToPostgres.ts similarity index 100% rename from packages/store-sync/src/postgres/syncToPostgres.ts rename to packages/store-sync/src/postgres-decoded/syncToPostgres.ts diff --git a/packages/store-sync/src/postgres/transformSchemaName.ts b/packages/store-sync/src/postgres-decoded/transformSchemaName.ts similarity index 100% rename from packages/store-sync/src/postgres/transformSchemaName.ts rename to packages/store-sync/src/postgres-decoded/transformSchemaName.ts From 4f23d527c03e0cf6476f7bfa87764bb7c425e29f Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 12:14:11 +0000 Subject: [PATCH 02/23] set up new default postgres indexer --- .../store-sync/src/postgres/cleanDatabase.ts | 12 + .../store-sync/src/postgres/columnTypes.ts | 74 ++++++ packages/store-sync/src/postgres/debug.ts | 3 + packages/store-sync/src/postgres/index.ts | 7 + packages/store-sync/src/postgres/pgDialect.ts | 10 + .../src/postgres/postgresStorage.test.ts | 82 +++++++ .../src/postgres/postgresStorage.ts | 222 ++++++++++++++++++ .../store-sync/src/postgres/schemaVersion.ts | 4 + .../src/postgres/setupTables.test.ts | 38 +++ .../store-sync/src/postgres/setupTables.ts | 76 ++++++ .../store-sync/src/postgres/syncToPostgres.ts | 51 ++++ packages/store-sync/src/postgres/tables.ts | 45 ++++ .../src/postgres/transformSchemaName.ts | 9 + 13 files changed, 633 insertions(+) create mode 100644 packages/store-sync/src/postgres/cleanDatabase.ts create mode 100644 packages/store-sync/src/postgres/columnTypes.ts create mode 100644 packages/store-sync/src/postgres/debug.ts create mode 100644 packages/store-sync/src/postgres/index.ts create mode 100644 packages/store-sync/src/postgres/pgDialect.ts create mode 100644 packages/store-sync/src/postgres/postgresStorage.test.ts create mode 100644 packages/store-sync/src/postgres/postgresStorage.ts create mode 100644 packages/store-sync/src/postgres/schemaVersion.ts create mode 100644 packages/store-sync/src/postgres/setupTables.test.ts create mode 100644 packages/store-sync/src/postgres/setupTables.ts create mode 100644 packages/store-sync/src/postgres/syncToPostgres.ts create mode 100644 packages/store-sync/src/postgres/tables.ts create mode 100644 packages/store-sync/src/postgres/transformSchemaName.ts diff --git a/packages/store-sync/src/postgres/cleanDatabase.ts b/packages/store-sync/src/postgres/cleanDatabase.ts new file mode 100644 index 0000000000..f0ccc355d6 --- /dev/null +++ b/packages/store-sync/src/postgres/cleanDatabase.ts @@ -0,0 +1,12 @@ +import { PgDatabase } from "drizzle-orm/pg-core"; +import { schemaName } from "./tables"; +import { debug } from "./debug"; +import { sql } from "drizzle-orm"; +import { pgDialect } from "./pgDialect"; + +// This intentionally just cleans up known schemas/tables/rows. We could drop the database but that's scary. + +export async function cleanDatabase(db: PgDatabase): Promise { + debug(`dropping schema ${schemaName} and all of its tables`); + await db.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); +} diff --git a/packages/store-sync/src/postgres/columnTypes.ts b/packages/store-sync/src/postgres/columnTypes.ts new file mode 100644 index 0000000000..da9d9761f8 --- /dev/null +++ b/packages/store-sync/src/postgres/columnTypes.ts @@ -0,0 +1,74 @@ +import { customType } from "drizzle-orm/pg-core"; +import superjson from "superjson"; +import { Address, ByteArray, bytesToHex, getAddress, Hex, hexToBytes } from "viem"; + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const asJson = (name: string) => + customType<{ data: TData; driverData: string }>({ + dataType() { + // TODO: move to json column type? if we do, we'll prob wanna choose something other than superjson since it adds one level of depth (json/meta keys) + return "text"; + }, + toDriver(data: TData): string { + return superjson.stringify(data); + }, + fromDriver(driverData: string): TData { + return superjson.parse(driverData); + }, + })(name); + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const asNumber = (name: string, columnType: string) => + customType<{ data: number; driverData: string }>({ + dataType() { + return columnType; + }, + toDriver(data: number): string { + return String(data); + }, + fromDriver(driverData: string): number { + return Number(driverData); + }, + })(name); + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const asBigInt = (name: string, columnType: string) => + customType<{ data: bigint; driverData: string }>({ + dataType() { + return columnType; + }, + toDriver(data: bigint): string { + return String(data); + }, + fromDriver(driverData: string): bigint { + return BigInt(driverData); + }, + })(name); + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const asHex = (name: string) => + customType<{ data: Hex; driverData: ByteArray }>({ + dataType() { + return "bytea"; + }, + toDriver(data: Hex): ByteArray { + return hexToBytes(data); + }, + fromDriver(driverData: ByteArray): Hex { + return bytesToHex(driverData); + }, + })(name); + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const asAddress = (name: string) => + customType<{ data: Address; driverData: ByteArray }>({ + dataType() { + return "bytea"; + }, + toDriver(data: Address): ByteArray { + return hexToBytes(data); + }, + fromDriver(driverData: ByteArray): Address { + return getAddress(bytesToHex(driverData)); + }, + })(name); diff --git a/packages/store-sync/src/postgres/debug.ts b/packages/store-sync/src/postgres/debug.ts new file mode 100644 index 0000000000..306f33b44b --- /dev/null +++ b/packages/store-sync/src/postgres/debug.ts @@ -0,0 +1,3 @@ +import { debug as parentDebug } from "../debug"; + +export const debug = parentDebug.extend("postgres"); diff --git a/packages/store-sync/src/postgres/index.ts b/packages/store-sync/src/postgres/index.ts new file mode 100644 index 0000000000..2a10e8d2f3 --- /dev/null +++ b/packages/store-sync/src/postgres/index.ts @@ -0,0 +1,7 @@ +export * from "./cleanDatabase"; +export * from "./columnTypes"; +export * from "./schemaVersion"; +export * from "./postgresStorage"; +export * from "./setupTables"; +export * from "./syncToPostgres"; +export * from "./tables"; diff --git a/packages/store-sync/src/postgres/pgDialect.ts b/packages/store-sync/src/postgres/pgDialect.ts new file mode 100644 index 0000000000..f5f42a526b --- /dev/null +++ b/packages/store-sync/src/postgres/pgDialect.ts @@ -0,0 +1,10 @@ +import { DummyDriver, Kysely, PostgresAdapter, PostgresIntrospector, PostgresQueryCompiler } from "kysely"; + +export const pgDialect = new Kysely({ + dialect: { + createAdapter: (): PostgresAdapter => new PostgresAdapter(), + createDriver: (): DummyDriver => new DummyDriver(), + createIntrospector: (db: Kysely): PostgresIntrospector => new PostgresIntrospector(db), + createQueryCompiler: (): PostgresQueryCompiler => new PostgresQueryCompiler(), + }, +}); diff --git a/packages/store-sync/src/postgres/postgresStorage.test.ts b/packages/store-sync/src/postgres/postgresStorage.test.ts new file mode 100644 index 0000000000..c50c4d342b --- /dev/null +++ b/packages/store-sync/src/postgres/postgresStorage.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { DefaultLogger, eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { Hex, RpcLog, createPublicClient, decodeEventLog, formatLog, http } from "viem"; +import { foundry } from "viem/chains"; +import { PostgresStorageAdapter, postgresStorage } from "./postgresStorage"; +import { groupLogsByBlockNumber } from "@latticexyz/block-logs-stream"; +import { storeEventsAbi } from "@latticexyz/store"; +import { StoreEventsLog } from "../common"; +import worldRpcLogs from "../../../../test-data/world-logs.json"; +import { chainTable, recordsTable } from "./tables"; +import { resourceToHex } from "@latticexyz/common"; + +const blocks = groupLogsByBlockNumber( + worldRpcLogs.map((log) => { + const { eventName, args } = decodeEventLog({ + abi: storeEventsAbi, + data: log.data as Hex, + topics: log.topics as [Hex, ...Hex[]], + strict: true, + }); + return formatLog(log as any as RpcLog, { args, eventName: eventName as string }) as StoreEventsLog; + }) +); + +describe("postgresStorage", async () => { + const db = drizzle(postgres(process.env.DATABASE_URL!), { + logger: new DefaultLogger(), + }); + + const publicClient = createPublicClient({ + chain: foundry, + transport: http(), + }); + + let storageAdapter: PostgresStorageAdapter; + + beforeEach(async () => { + storageAdapter = await postgresStorage({ database: db, publicClient }); + return storageAdapter.cleanUp; + }); + + it("should create tables and data from block log", async () => { + for (const block of blocks) { + await storageAdapter.storageAdapter(block); + } + + expect(await db.select().from(chainTable)).toMatchInlineSnapshot(` + [ + { + "chainId": 31337, + "lastUpdatedBlockNumber": 12n, + }, + ] + `); + + expect( + await db + .select() + .from(recordsTable) + .where(eq(recordsTable.tableId, resourceToHex({ type: "table", namespace: "", name: "NumberList" }))) + ).toMatchInlineSnapshot(` + [ + { + "address": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342", + "dynamicData": "0x000001a400000045", + "encodedLengths": "0x0000000000000000000000000000000000000000000000000800000000000008", + "isDeleted": false, + "key0": null, + "key1": null, + "keyBytes": "0x", + "lastUpdatedBlockNumber": 12n, + "staticData": null, + "tableId": "0x746200000000000000000000000000004e756d6265724c697374000000000000", + }, + ] + `); + + await storageAdapter.cleanUp(); + }); +}); diff --git a/packages/store-sync/src/postgres/postgresStorage.ts b/packages/store-sync/src/postgres/postgresStorage.ts new file mode 100644 index 0000000000..424b6e68c7 --- /dev/null +++ b/packages/store-sync/src/postgres/postgresStorage.ts @@ -0,0 +1,222 @@ +import { PublicClient, encodePacked, size } from "viem"; +import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; +import { and, eq } from "drizzle-orm"; +import { StoreConfig } from "@latticexyz/store"; +import { debug } from "./debug"; +import { chainTable, storesTable, recordsTable } from "./tables"; +import { spliceHex } from "@latticexyz/common"; +import { setupTables } from "./setupTables"; +import { StorageAdapter, StorageAdapterBlock } from "../common"; + +const tables = [chainTable, storesTable, recordsTable] as const; + +// Currently assumes one DB per chain ID + +export type PostgresStorageAdapter = { + storageAdapter: StorageAdapter; + cleanUp: () => Promise; +}; + +export async function postgresStorage({ + database, + publicClient, +}: { + database: PgDatabase; + publicClient: PublicClient; + config?: TConfig; +}): Promise { + const cleanUp: (() => Promise)[] = []; + + const chainId = publicClient.chain?.id ?? (await publicClient.getChainId()); + + cleanUp.push(await setupTables(database, Object.values(tables))); + + async function postgresStorageAdapter({ blockNumber, logs }: StorageAdapterBlock): Promise { + await database.transaction(async (tx) => { + // TODO: update stores table + + for (const log of logs) { + debug(log.eventName, log); + + const keyBytes = encodePacked(["bytes32[]"], [log.args.keyTuple]); + + if (log.eventName === "Store_SetRecord") { + await tx + .insert(recordsTable) + .values({ + address: log.address, + tableId: log.args.tableId, + keyBytes, + key0: log.args.keyTuple[0], + key1: log.args.keyTuple[1], + staticData: log.args.staticData, + encodedLengths: log.args.encodedLengths, + dynamicData: log.args.dynamicData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }) + .onConflictDoUpdate({ + target: [recordsTable.address, recordsTable.tableId, recordsTable.keyBytes], + set: { + staticData: log.args.staticData, + encodedLengths: log.args.encodedLengths, + dynamicData: log.args.dynamicData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }, + }) + .execute(); + } else if (log.eventName === "Store_SpliceStaticData") { + // TODO: replace this operation with SQL `overlay()` (https://www.postgresql.org/docs/9.3/functions-binarystring.html) + + const previousValue = await tx + .select({ staticData: recordsTable.staticData }) + .from(recordsTable) + .where( + and( + eq(recordsTable.address, log.address), + eq(recordsTable.tableId, log.args.tableId), + eq(recordsTable.keyBytes, keyBytes) + ) + ) + .limit(1) + .execute() + .then((rows) => rows.find(() => true)); + + const previousStaticData = previousValue?.staticData ?? "0x"; + const newStaticData = spliceHex(previousStaticData, log.args.start, size(log.args.data), log.args.data); + + debug("upserting record via splice static", { + address: log.address, + tableId: log.args.tableId, + keyTuple: log.args.keyTuple, + previousStaticData, + newStaticData, + previousValue, + }); + + await tx + .insert(recordsTable) + .values({ + address: log.address, + tableId: log.args.tableId, + keyBytes, + key0: log.args.keyTuple[0], + key1: log.args.keyTuple[1], + staticData: newStaticData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }) + .onConflictDoUpdate({ + target: [recordsTable.address, recordsTable.tableId, recordsTable.keyBytes], + set: { + staticData: newStaticData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }, + }) + .execute(); + } else if (log.eventName === "Store_SpliceDynamicData") { + // TODO: replace this operation with SQL `overlay()` (https://www.postgresql.org/docs/9.3/functions-binarystring.html) + + const previousValue = await tx + .select({ dynamicData: recordsTable.dynamicData }) + .from(recordsTable) + .where( + and( + eq(recordsTable.address, log.address), + eq(recordsTable.tableId, log.args.tableId), + eq(recordsTable.keyBytes, keyBytes) + ) + ) + .limit(1) + .execute() + .then((rows) => rows.find(() => true)); + + const previousDynamicData = previousValue?.dynamicData ?? "0x"; + const newDynamicData = spliceHex(previousDynamicData, log.args.start, log.args.deleteCount, log.args.data); + + debug("upserting record via splice dynamic", { + address: log.address, + tableId: log.args.tableId, + keyTuple: log.args.keyTuple, + previousDynamicData, + newDynamicData, + previousValue, + }); + + await tx + .insert(recordsTable) + .values({ + address: log.address, + tableId: log.args.tableId, + keyBytes, + key0: log.args.keyTuple[0], + key1: log.args.keyTuple[1], + encodedLengths: log.args.encodedLengths, + dynamicData: newDynamicData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }) + .onConflictDoUpdate({ + target: [recordsTable.address, recordsTable.tableId, recordsTable.keyBytes], + set: { + encodedLengths: log.args.encodedLengths, + dynamicData: newDynamicData, + lastUpdatedBlockNumber: blockNumber, + isDeleted: false, + }, + }) + .execute(); + } else if (log.eventName === "Store_DeleteRecord") { + debug("deleting record", { + address: log.address, + tableId: log.args.tableId, + keyTuple: log.args.keyTuple, + }); + + await tx + .update(recordsTable) + .set({ + staticData: null, + encodedLengths: null, + dynamicData: null, + lastUpdatedBlockNumber: blockNumber, + isDeleted: true, + }) + .where( + and( + eq(recordsTable.address, log.address), + eq(recordsTable.tableId, log.args.tableId), + eq(recordsTable.keyBytes, keyBytes) + ) + ) + .execute(); + } + } + + await tx + .insert(chainTable) + .values({ + chainId, + lastUpdatedBlockNumber: blockNumber, + }) + .onConflictDoUpdate({ + target: [chainTable.chainId], + set: { + lastUpdatedBlockNumber: blockNumber, + }, + }) + .execute(); + }); + } + + return { + storageAdapter: postgresStorageAdapter, + cleanUp: async (): Promise => { + for (const fn of cleanUp) { + await fn(); + } + }, + }; +} diff --git a/packages/store-sync/src/postgres/schemaVersion.ts b/packages/store-sync/src/postgres/schemaVersion.ts new file mode 100644 index 0000000000..397579c6b3 --- /dev/null +++ b/packages/store-sync/src/postgres/schemaVersion.ts @@ -0,0 +1,4 @@ +// When this is incremented, it forces all indexers to reindex from scratch the next time they start up. +// Only use this when the schemas change, until we get proper schema migrations. +// TODO: instead of this, detect schema changes and drop/recreate tables as needed +export const schemaVersion = 1; diff --git a/packages/store-sync/src/postgres/setupTables.test.ts b/packages/store-sync/src/postgres/setupTables.test.ts new file mode 100644 index 0000000000..9cedad1fb1 --- /dev/null +++ b/packages/store-sync/src/postgres/setupTables.test.ts @@ -0,0 +1,38 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { chainTable, storesTable, recordsTable } from "./tables"; +import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; +import { DefaultLogger } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { setupTables } from "./setupTables"; + +describe("setupTables", async () => { + let db: PgDatabase; + + beforeEach(async () => { + db = drizzle(postgres(process.env.DATABASE_URL!), { + logger: new DefaultLogger(), + }); + }); + + describe("before running", () => { + it("should be missing schemas", async () => { + await expect(db.select().from(chainTable)).rejects.toThrow(/relation "\w+mud.chain" does not exist/); + await expect(db.select().from(storesTable)).rejects.toThrow(/relation "\w+mud.stores" does not exist/); + await expect(db.select().from(recordsTable)).rejects.toThrow(/relation "\w+mud.records" does not exist/); + }); + }); + + describe("after running", () => { + beforeEach(async () => { + const cleanUp = await setupTables(db, Object.values([chainTable, storesTable, recordsTable])); + return cleanUp; + }); + + it("should have schemas", async () => { + expect(await db.select().from(chainTable)).toMatchInlineSnapshot("[]"); + expect(await db.select().from(storesTable)).toMatchInlineSnapshot("[]"); + expect(await db.select().from(recordsTable)).toMatchInlineSnapshot("[]"); + }); + }); +}); diff --git a/packages/store-sync/src/postgres/setupTables.ts b/packages/store-sync/src/postgres/setupTables.ts new file mode 100644 index 0000000000..f3f6c00c79 --- /dev/null +++ b/packages/store-sync/src/postgres/setupTables.ts @@ -0,0 +1,76 @@ +import { AnyPgColumn, PgTableWithColumns, PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; +import { getTableColumns, sql } from "drizzle-orm"; +import { ColumnDataType } from "kysely"; +import { isDefined } from "@latticexyz/common/utils"; +import { debug } from "./debug"; +import { pgDialect } from "./pgDialect"; + +export async function setupTables( + db: PgDatabase, + tables: PgTableWithColumns[] +): Promise<() => Promise> { + const schemaNames = [...new Set(tables.map((table) => getTableConfig(table).schema).filter(isDefined))]; + + await db.transaction(async (tx) => { + for (const schemaName of schemaNames) { + debug(`creating namespace ${schemaName}`); + await tx.execute(sql.raw(pgDialect.schema.createSchema(schemaName).ifNotExists().compile().sql)); + } + + for (const table of tables) { + const tableConfig = getTableConfig(table); + const scopedDb = tableConfig.schema ? pgDialect.withSchema(tableConfig.schema) : pgDialect; + + let query = scopedDb.schema.createTable(tableConfig.name).ifNotExists(); + + const columns = Object.values(getTableColumns(table)) as AnyPgColumn[]; + for (const column of columns) { + query = query.addColumn(column.name, column.getSQLType() as ColumnDataType, (col) => { + if (column.notNull) { + col = col.notNull(); + } + if (column.hasDefault && typeof column.default !== "undefined") { + col = col.defaultTo(column.default); + } + return col; + }); + } + + const primaryKeyColumns = columns.filter((column) => column.primary).map((column) => column.name); + if (primaryKeyColumns.length) { + query = query.addPrimaryKeyConstraint(`${primaryKeyColumns.join("_")}_pk`, primaryKeyColumns as any); + } + + for (const pk of tableConfig.primaryKeys) { + query = query.addPrimaryKeyConstraint(pk.getName(), pk.columns.map((col) => col.name) as any); + } + + debug(`creating table ${tableConfig.name} in namespace ${tableConfig.schema}`); + await tx.execute(sql.raw(query.compile().sql)); + + for (const index of tableConfig.indexes) { + const columnNames = index.config.columns.map((col) => col.name); + let query = scopedDb.schema + .createIndex(index.config.name ?? `${columnNames.join("_")}_index`) + .on(tableConfig.name) + .columns(columnNames) + .ifNotExists(); + if (index.config.unique) { + query = query.unique(); + } + await tx.execute(sql.raw(query.compile().sql)); + } + } + }); + + return async () => { + for (const schemaName of schemaNames) { + try { + debug(`dropping namespace ${schemaName} and all of its tables`); + await db.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); + } catch (error) { + debug(`failed to drop namespace ${schemaName}`, error); + } + } + }; +} diff --git a/packages/store-sync/src/postgres/syncToPostgres.ts b/packages/store-sync/src/postgres/syncToPostgres.ts new file mode 100644 index 0000000000..10ea4cf2af --- /dev/null +++ b/packages/store-sync/src/postgres/syncToPostgres.ts @@ -0,0 +1,51 @@ +import { StoreConfig } from "@latticexyz/store"; +import { PgDatabase } from "drizzle-orm/pg-core"; +import { SyncOptions, SyncResult } from "../common"; +import { postgresStorage } from "./postgresStorage"; +import { createStoreSync } from "../createStoreSync"; + +type SyncToPostgresOptions = SyncOptions & { + /** + * [Postgres database object from Drizzle][0]. + * + * [0]: https://orm.drizzle.team/docs/installation-and-db-connection/postgresql/postgresjs + */ + database: PgDatabase; + startSync?: boolean; +}; + +type SyncToPostgresResult = SyncResult & { + stopSync: () => void; +}; + +/** + * Creates an indexer to process and store blockchain events. + * + * @param {CreateIndexerOptions} options See `CreateIndexerOptions`. + * @returns A function to unsubscribe from the block stream, effectively stopping the indexer. + */ +export async function syncToPostgres({ + config, + database, + publicClient, + startSync = true, + ...syncOptions +}: SyncToPostgresOptions): Promise { + const { storageAdapter } = await postgresStorage({ database, publicClient, config }); + const storeSync = await createStoreSync({ + storageAdapter, + config, + publicClient, + ...syncOptions, + }); + + const sub = startSync ? storeSync.storedBlockLogs$.subscribe() : null; + const stopSync = (): void => { + sub?.unsubscribe(); + }; + + return { + ...storeSync, + stopSync, + }; +} diff --git a/packages/store-sync/src/postgres/tables.ts b/packages/store-sync/src/postgres/tables.ts new file mode 100644 index 0000000000..d7183ae0d5 --- /dev/null +++ b/packages/store-sync/src/postgres/tables.ts @@ -0,0 +1,45 @@ +import { boolean, index, pgSchema, primaryKey, text } from "drizzle-orm/pg-core"; +import { transformSchemaName } from "./transformSchemaName"; +import { asAddress, asBigInt, asHex, asNumber } from "./columnTypes"; + +export const schemaName = transformSchemaName("mud"); + +/** + * Singleton table for the state of the chain we're indexing + */ +export const chainTable = pgSchema(schemaName).table("chain", { + chainId: asNumber("chain_id", "bigint").notNull().primaryKey(), + lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), +}); + +export const storesTable = pgSchema(schemaName).table("stores", { + address: asAddress("address").notNull().primaryKey(), + storeVersion: text("store_version").notNull(), + lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), +}); + +export const recordsTable = pgSchema(schemaName).table( + "records", + { + address: asAddress("address").notNull(), + tableId: asHex("table_id").notNull(), + /** + * `keyBytes` is equivalent to `abi.encodePacked(bytes32[] keyTuple)` + */ + keyBytes: asHex("key_bytes").notNull(), + key0: asHex("key0"), + key1: asHex("key1"), + lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), + staticData: asHex("static_data"), + encodedLengths: asHex("encoded_lengths"), + dynamicData: asHex("dynamic_data"), + isDeleted: boolean("is_deleted"), + }, + (table) => ({ + pk: primaryKey(table.address, table.tableId, table.keyBytes), + key0Index: index("key0_index").on(table.address, table.tableId, table.key0), + key1Index: index("key1_index").on(table.address, table.tableId, table.key1), + // TODO: add indices for querying without table ID + // TODO: add indices for querying multiple keys + }) +); diff --git a/packages/store-sync/src/postgres/transformSchemaName.ts b/packages/store-sync/src/postgres/transformSchemaName.ts new file mode 100644 index 0000000000..353d2c1632 --- /dev/null +++ b/packages/store-sync/src/postgres/transformSchemaName.ts @@ -0,0 +1,9 @@ +/** + * Helps parallelize creating/altering tables in tests + */ +export function transformSchemaName(schemaName: string): string { + if (process.env.NODE_ENV === "test") { + return `${process.pid}_${process.env.VITEST_POOL_ID}__${schemaName}`; + } + return schemaName; +} From e38a0aa29275dbd5bd992d658bf1303f090ce189 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 12:14:58 +0000 Subject: [PATCH 03/23] add new build for decoded postgres --- packages/store-sync/package.json | 4 ++++ packages/store-sync/tsup.config.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/packages/store-sync/package.json b/packages/store-sync/package.json index abd1c2ea5d..e85e8b6481 100644 --- a/packages/store-sync/package.json +++ b/packages/store-sync/package.json @@ -12,6 +12,7 @@ "exports": { ".": "./dist/index.js", "./postgres": "./dist/postgres/index.js", + "./postgres-decoded": "./dist/postgres-decoded/index.js", "./recs": "./dist/recs/index.js", "./sqlite": "./dist/sqlite/index.js", "./trpc-indexer": "./dist/trpc-indexer/index.js", @@ -25,6 +26,9 @@ "postgres": [ "./src/postgres/index.ts" ], + "postgres-decoded": [ + "./src/postgres-decoded/index.ts" + ], "recs": [ "./src/recs/index.ts" ], diff --git a/packages/store-sync/tsup.config.ts b/packages/store-sync/tsup.config.ts index e2345f3a5b..15719d50b0 100644 --- a/packages/store-sync/tsup.config.ts +++ b/packages/store-sync/tsup.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ "src/index.ts", "src/sqlite/index.ts", "src/postgres/index.ts", + "src/postgres-decoded/index.ts", "src/recs/index.ts", "src/trpc-indexer/index.ts", "src/zustand/index.ts", From 9b76bcf940ed4c48f5916f3ebad19385f52e9f64 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 15:48:49 +0000 Subject: [PATCH 04/23] update postgres indexer, add new getLogs endpoint --- .../store-indexer/bin/postgres-indexer.ts | 35 +++----- packages/store-indexer/package.json | 3 +- .../src/postgres/createQueryAdapter.ts | 88 ++++++------------- .../store-indexer/src/postgres/getLogs.ts | 69 +++++++++++++++ .../src/sqlite/createQueryAdapter.ts | 4 + packages/store-sync/src/index.ts | 2 + .../store-sync/src/isTableRegistrationLog.ts | 3 + packages/store-sync/src/logToTable.ts | 3 + ...e.test.ts => createStorageAdapter.test.ts} | 6 +- ...gresStorage.ts => createStorageAdapter.ts} | 10 ++- packages/store-sync/src/postgres/index.ts | 2 +- .../store-sync/src/postgres/syncToPostgres.ts | 4 +- packages/store-sync/src/postgres/tables.ts | 2 +- packages/store-sync/src/tableToLog.ts | 3 + .../store-sync/src/trpc-indexer/common.ts | 10 ++- .../src/trpc-indexer/createAppRouter.ts | 22 +++++ pnpm-lock.yaml | 14 --- 17 files changed, 169 insertions(+), 111 deletions(-) create mode 100644 packages/store-indexer/src/postgres/getLogs.ts rename packages/store-sync/src/postgres/{postgresStorage.test.ts => createStorageAdapter.test.ts} (91%) rename packages/store-sync/src/postgres/{postgresStorage.ts => createStorageAdapter.ts} (97%) diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index b6a288d5d1..946f342376 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -7,7 +7,7 @@ import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; -import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres"; +import { cleanDatabase, createStorageAdapter, schemaVersion } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; import { indexerEnvSchema, parseEnv } from "./parseEnv"; @@ -37,37 +37,26 @@ const publicClient = createPublicClient({ const chainId = await publicClient.getChainId(); const database = drizzle(postgres(env.DATABASE_URL)); -const { storageAdapter, internalTables } = await postgresStorage({ database, publicClient }); +const { storageAdapter, tables } = await createStorageAdapter({ database, publicClient }); let startBlock = env.START_BLOCK; // Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error. +// TODO: query if the DB exists instead of try/catch try { - const currentChainStates = await database + const chainState = await database .select() - .from(internalTables.chain) - .where(eq(internalTables.chain.chainId, chainId)) - .execute(); - // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212) - const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0]; + .from(tables.chainTable) + .where(eq(tables.chainTable.chainId, chainId)) + .execute() + .then((rows) => rows.find(() => true)); - if (currentChainState != null) { - if (currentChainState.schemaVersion != schemaVersion) { - console.log( - "schema version changed from", - currentChainState.schemaVersion, - "to", - schemaVersion, - "cleaning database" - ); - await cleanDatabase(database); - } else if (currentChainState.lastUpdatedBlockNumber != null) { - console.log("resuming from block number", currentChainState.lastUpdatedBlockNumber + 1n); - startBlock = currentChainState.lastUpdatedBlockNumber + 1n; - } + if (chainState?.lastUpdatedBlockNumber != null) { + startBlock = chainState.lastUpdatedBlockNumber + 1n; + console.log("resuming from block number", startBlock); } } catch (error) { - // ignore errors, this is optional + // ignore errors for now } const { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({ diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json index 051bc8fd3a..e1c08d0ba8 100644 --- a/packages/store-indexer/package.json +++ b/packages/store-indexer/package.json @@ -27,7 +27,7 @@ "lint": "eslint .", "start:postgres": "concurrently -n indexer,frontend -c cyan,magenta 'tsx bin/postgres-indexer' 'tsx bin/postgres-frontend'", "start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres", - "start:postgres:testnet": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres", + "start:postgres:testnet": "DEBUG=mud:* DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres", "start:sqlite": "tsx bin/sqlite-indexer", "start:sqlite:local": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite", "start:sqlite:testnet": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:sqlite", @@ -43,7 +43,6 @@ "@latticexyz/store-sync": "workspace:*", "@trpc/client": "10.34.0", "@trpc/server": "10.34.0", - "@wagmi/chains": "^0.2.22", "better-sqlite3": "^8.6.0", "debug": "^4.3.4", "dotenv": "^16.0.3", diff --git a/packages/store-indexer/src/postgres/createQueryAdapter.ts b/packages/store-indexer/src/postgres/createQueryAdapter.ts index aced9d193b..1e60f26f65 100644 --- a/packages/store-indexer/src/postgres/createQueryAdapter.ts +++ b/packages/store-indexer/src/postgres/createQueryAdapter.ts @@ -1,10 +1,10 @@ -import { eq, getTableName } from "drizzle-orm"; import { PgDatabase } from "drizzle-orm/pg-core"; -import { buildTable, buildInternalTables, getTables } from "@latticexyz/store-sync/postgres"; import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer"; import { debug } from "../debug"; import { getAddress } from "viem"; -import { decodeDynamicField } from "@latticexyz/protocol-parser"; +import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; +import { getLogs } from "./getLogs"; +import { TableWithRecords, isTableRegistrationLog, logToTable } from "@latticexyz/store-sync"; /** * Creates a query adapter for the tRPC server/client to query data from Postgres. @@ -14,70 +14,34 @@ import { decodeDynamicField } from "@latticexyz/protocol-parser"; */ export async function createQueryAdapter(database: PgDatabase): Promise { const adapter: QueryAdapter = { - async findAll({ chainId, address, filters = [] }) { - // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters. - // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily) - const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId))); - const tables = (await getTables(database)) - .filter((table) => address == null || getAddress(address) === getAddress(table.address)) - .filter((table) => !tableIds.length || tableIds.includes(table.tableId)); + async getLogs(opts) { + return getLogs(database, opts); + }, + async findAll(opts) { + const { blockNumber, logs } = await getLogs(database, opts); - const tablesWithRecords = await Promise.all( - tables.map(async (table) => { - const sqlTable = buildTable(table); - const records = await database - .select() - .from(sqlTable) - .where(eq(sqlTable.__isDeleted, false)) - .execute() - // Apparently sometimes we can have tables that exist in the internal table but no relation/schema set up, so queries fail. - // See https://github.com/latticexyz/mud/issues/1923 - // TODO: make a more robust fix for this - .catch((error) => { - console.error( - "Could not query for records, returning empty set for table", - getTableName(sqlTable), - error - ); - return []; - }); - const filteredRecords = !filters.length - ? records - : records.filter((record) => { - const keyTuple = decodeDynamicField("bytes32[]", record.__key); - return filters.some( - (filter) => - filter.tableId === table.tableId && - (filter.key0 == null || filter.key0 === keyTuple[0]) && - (filter.key1 == null || filter.key1 === keyTuple[1]) - ); - }); - return { - ...table, - records: filteredRecords.map((record) => ({ - key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])), - value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])), - })), - }; - }) - ); + const tables = logs.filter(isTableRegistrationLog).map(logToTable); - const internalTables = buildInternalTables(); - const metadata = await database - .select() - .from(internalTables.chain) - .where(eq(internalTables.chain.chainId, chainId)) - .execute(); - const { lastUpdatedBlockNumber } = metadata[0] ?? {}; + const tablesWithRecords: TableWithRecords[] = tables.map((table) => { + const records = logs + .filter((log) => getAddress(log.address) === getAddress(table.address) && log.args.tableId === table.tableId) + .map((log) => ({ + key: decodeKey(table.keySchema, log.args.keyTuple), + value: decodeValueArgs(table.valueSchema, log.args), + })); - const result = { - blockNumber: lastUpdatedBlockNumber ?? null, - tables: tablesWithRecords, - }; + return { + ...table, + records, + }; + }); - debug("findAll", chainId, address, result); + debug("findAll: decoded %d logs across %d tables", logs.length, tables.length); - return result; + return { + blockNumber, + tables: tablesWithRecords, + }; }, }; return adapter; diff --git a/packages/store-indexer/src/postgres/getLogs.ts b/packages/store-indexer/src/postgres/getLogs.ts new file mode 100644 index 0000000000..4479f77fc3 --- /dev/null +++ b/packages/store-indexer/src/postgres/getLogs.ts @@ -0,0 +1,69 @@ +import { PgDatabase } from "drizzle-orm/pg-core"; +import { Hex } from "viem"; +import { StorageAdapterLog, SyncFilter } from "@latticexyz/store-sync"; +import { chainTable, recordsTable } from "@latticexyz/store-sync/postgres"; +import { and, eq, or } from "drizzle-orm"; +import { decodeDynamicField } from "@latticexyz/protocol-parser"; +import { bigIntMax } from "@latticexyz/common/utils"; + +export async function getLogs( + database: PgDatabase, + { + chainId, + address, + filters = [], + }: { + readonly chainId: number; + readonly address?: Hex; + readonly filters?: readonly SyncFilter[]; + } +): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: "Store_SetRecord" })[] }> { + const conditions = filters.length + ? filters.map((filter) => + and( + address != null ? eq(recordsTable.address, address) : undefined, + eq(recordsTable.tableId, filter.tableId), + filter.key0 != null ? eq(recordsTable.key0, filter.key0) : undefined, + filter.key1 != null ? eq(recordsTable.key1, filter.key1) : undefined + ) + ) + : address != null + ? [eq(recordsTable.address, address)] + : []; + + const chainState = await database + .select() + .from(chainTable) + .where(eq(chainTable.chainId, chainId)) + .execute() + .then((rows) => rows.find(() => true)); + let blockNumber = chainState?.lastUpdatedBlockNumber ?? 0n; + + const records = await database + .select() + .from(recordsTable) + .where(or(...conditions)); + blockNumber = bigIntMax( + blockNumber, + records.reduce((max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n), 0n) + ); + + const logs = records + .filter((record) => !record.isDeleted) + .map( + (record) => + ({ + address: record.address, + eventName: "Store_SetRecord", + args: { + tableId: record.tableId, + keyTuple: decodeDynamicField("bytes32[]", record.keyBytes), + staticData: record.staticData ?? "0x", + encodedLengths: record.encodedLengths ?? "0x", + dynamicData: record.dynamicData ?? "0x", + }, + } as const) + ); + + return { blockNumber, logs }; +} diff --git a/packages/store-indexer/src/sqlite/createQueryAdapter.ts b/packages/store-indexer/src/sqlite/createQueryAdapter.ts index aee529e4f1..8d68d5f7df 100644 --- a/packages/store-indexer/src/sqlite/createQueryAdapter.ts +++ b/packages/store-indexer/src/sqlite/createQueryAdapter.ts @@ -14,6 +14,10 @@ import { decodeDynamicField } from "@latticexyz/protocol-parser"; */ export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise { const adapter: QueryAdapter = { + async getLogs(opts) { + // TODO + throw new Error("Not implemented"); + }, async findAll({ chainId, address, filters = [] }) { // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters. // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily) diff --git a/packages/store-sync/src/index.ts b/packages/store-sync/src/index.ts index 2e0771488d..deb192e47d 100644 --- a/packages/store-sync/src/index.ts +++ b/packages/store-sync/src/index.ts @@ -1,3 +1,5 @@ export * from "./common"; export * from "./createStoreSync"; export * from "./SyncStep"; +export * from "./isTableRegistrationLog"; +export * from "./logToTable"; diff --git a/packages/store-sync/src/isTableRegistrationLog.ts b/packages/store-sync/src/isTableRegistrationLog.ts index c71ad1f3e4..8f6b6148d6 100644 --- a/packages/store-sync/src/isTableRegistrationLog.ts +++ b/packages/store-sync/src/isTableRegistrationLog.ts @@ -1,5 +1,8 @@ import { StorageAdapterLog, storeTables } from "./common"; +/** + * @internal + */ export function isTableRegistrationLog( log: StorageAdapterLog ): log is StorageAdapterLog & { eventName: "Store_SetRecord" } { diff --git a/packages/store-sync/src/logToTable.ts b/packages/store-sync/src/logToTable.ts index 0572c6f3f0..0d50dce3aa 100644 --- a/packages/store-sync/src/logToTable.ts +++ b/packages/store-sync/src/logToTable.ts @@ -3,6 +3,9 @@ import { Hex, concatHex, decodeAbiParameters, parseAbiParameters } from "viem"; import { StorageAdapterLog, Table, schemasTable } from "./common"; import { hexToResource } from "@latticexyz/common"; +/** + * @internal + */ export function logToTable(log: StorageAdapterLog & { eventName: "Store_SetRecord" }): Table { const [tableId, ...otherKeys] = log.args.keyTuple; if (otherKeys.length) { diff --git a/packages/store-sync/src/postgres/postgresStorage.test.ts b/packages/store-sync/src/postgres/createStorageAdapter.test.ts similarity index 91% rename from packages/store-sync/src/postgres/postgresStorage.test.ts rename to packages/store-sync/src/postgres/createStorageAdapter.test.ts index c50c4d342b..83e02c7f98 100644 --- a/packages/store-sync/src/postgres/postgresStorage.test.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.test.ts @@ -4,7 +4,7 @@ import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { Hex, RpcLog, createPublicClient, decodeEventLog, formatLog, http } from "viem"; import { foundry } from "viem/chains"; -import { PostgresStorageAdapter, postgresStorage } from "./postgresStorage"; +import { PostgresStorageAdapter, createStorageAdapter } from "./createStorageAdapter"; import { groupLogsByBlockNumber } from "@latticexyz/block-logs-stream"; import { storeEventsAbi } from "@latticexyz/store"; import { StoreEventsLog } from "../common"; @@ -24,7 +24,7 @@ const blocks = groupLogsByBlockNumber( }) ); -describe("postgresStorage", async () => { +describe("createStorageAdapter", async () => { const db = drizzle(postgres(process.env.DATABASE_URL!), { logger: new DefaultLogger(), }); @@ -37,7 +37,7 @@ describe("postgresStorage", async () => { let storageAdapter: PostgresStorageAdapter; beforeEach(async () => { - storageAdapter = await postgresStorage({ database: db, publicClient }); + storageAdapter = await createStorageAdapter({ database: db, publicClient }); return storageAdapter.cleanUp; }); diff --git a/packages/store-sync/src/postgres/postgresStorage.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts similarity index 97% rename from packages/store-sync/src/postgres/postgresStorage.ts rename to packages/store-sync/src/postgres/createStorageAdapter.ts index 424b6e68c7..9cfbf6deab 100644 --- a/packages/store-sync/src/postgres/postgresStorage.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -8,16 +8,21 @@ import { spliceHex } from "@latticexyz/common"; import { setupTables } from "./setupTables"; import { StorageAdapter, StorageAdapterBlock } from "../common"; -const tables = [chainTable, storesTable, recordsTable] as const; +const tables = { + chainTable, + storesTable, + recordsTable, +} as const; // Currently assumes one DB per chain ID export type PostgresStorageAdapter = { storageAdapter: StorageAdapter; + tables: typeof tables; cleanUp: () => Promise; }; -export async function postgresStorage({ +export async function createStorageAdapter({ database, publicClient, }: { @@ -213,6 +218,7 @@ export async function postgresStorage return { storageAdapter: postgresStorageAdapter, + tables, cleanUp: async (): Promise => { for (const fn of cleanUp) { await fn(); diff --git a/packages/store-sync/src/postgres/index.ts b/packages/store-sync/src/postgres/index.ts index 2a10e8d2f3..ebd661611e 100644 --- a/packages/store-sync/src/postgres/index.ts +++ b/packages/store-sync/src/postgres/index.ts @@ -1,7 +1,7 @@ export * from "./cleanDatabase"; export * from "./columnTypes"; +export * from "./createStorageAdapter"; export * from "./schemaVersion"; -export * from "./postgresStorage"; export * from "./setupTables"; export * from "./syncToPostgres"; export * from "./tables"; diff --git a/packages/store-sync/src/postgres/syncToPostgres.ts b/packages/store-sync/src/postgres/syncToPostgres.ts index 10ea4cf2af..b4c98627e7 100644 --- a/packages/store-sync/src/postgres/syncToPostgres.ts +++ b/packages/store-sync/src/postgres/syncToPostgres.ts @@ -1,7 +1,7 @@ import { StoreConfig } from "@latticexyz/store"; import { PgDatabase } from "drizzle-orm/pg-core"; import { SyncOptions, SyncResult } from "../common"; -import { postgresStorage } from "./postgresStorage"; +import { createStorageAdapter } from "./createStorageAdapter"; import { createStoreSync } from "../createStoreSync"; type SyncToPostgresOptions = SyncOptions & { @@ -31,7 +31,7 @@ export async function syncToPostgres( startSync = true, ...syncOptions }: SyncToPostgresOptions): Promise { - const { storageAdapter } = await postgresStorage({ database, publicClient, config }); + const { storageAdapter } = await createStorageAdapter({ database, publicClient, config }); const storeSync = await createStoreSync({ storageAdapter, config, diff --git a/packages/store-sync/src/postgres/tables.ts b/packages/store-sync/src/postgres/tables.ts index d7183ae0d5..06a590b2a1 100644 --- a/packages/store-sync/src/postgres/tables.ts +++ b/packages/store-sync/src/postgres/tables.ts @@ -29,11 +29,11 @@ export const recordsTable = pgSchema(schemaName).table( keyBytes: asHex("key_bytes").notNull(), key0: asHex("key0"), key1: asHex("key1"), - lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), staticData: asHex("static_data"), encodedLengths: asHex("encoded_lengths"), dynamicData: asHex("dynamic_data"), isDeleted: boolean("is_deleted"), + lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), }, (table) => ({ pk: primaryKey(table.address, table.tableId, table.keyBytes), diff --git a/packages/store-sync/src/tableToLog.ts b/packages/store-sync/src/tableToLog.ts index 67e13f4b4b..ec1670101e 100644 --- a/packages/store-sync/src/tableToLog.ts +++ b/packages/store-sync/src/tableToLog.ts @@ -9,6 +9,9 @@ import { encodeAbiParameters, parseAbiParameters } from "viem"; import { StorageAdapterLog, Table, storeTables } from "./common"; import { flattenSchema } from "./flattenSchema"; +/** + * @internal + */ export function tableToLog(table: Table): StorageAdapterLog & { eventName: "Store_SetRecord" } { return { eventName: "Store_SetRecord", diff --git a/packages/store-sync/src/trpc-indexer/common.ts b/packages/store-sync/src/trpc-indexer/common.ts index 731c940362..eee07715a3 100644 --- a/packages/store-sync/src/trpc-indexer/common.ts +++ b/packages/store-sync/src/trpc-indexer/common.ts @@ -1,9 +1,17 @@ import { Hex } from "viem"; -import { SyncFilter, TableWithRecords } from "../common"; +import { StorageAdapterBlock, SyncFilter, TableWithRecords } from "../common"; export type QueryAdapter = { + /** + * @deprecated + */ findAll: (opts: { chainId: number; address?: Hex; filters?: SyncFilter[] }) => Promise<{ blockNumber: bigint | null; tables: TableWithRecords[]; }>; + getLogs: (opts: { + readonly chainId: number; + readonly address?: Hex; + readonly filters?: readonly SyncFilter[]; + }) => Promise; }; diff --git a/packages/store-sync/src/trpc-indexer/createAppRouter.ts b/packages/store-sync/src/trpc-indexer/createAppRouter.ts index 4d9272e61d..9ed0bd8413 100644 --- a/packages/store-sync/src/trpc-indexer/createAppRouter.ts +++ b/packages/store-sync/src/trpc-indexer/createAppRouter.ts @@ -11,6 +11,28 @@ export function createAppRouter() { }); return t.router({ + getRecords: t.procedure + .input( + z.object({ + chainId: z.number(), + address: z.string().refine(isHex).optional(), + filters: z + .array( + z.object({ + tableId: z.string().refine(isHex), + key0: z.string().refine(isHex).optional(), + key1: z.string().refine(isHex).optional(), + }) + ) + .optional(), + }) + ) + .query(async (opts): ReturnType => { + const { queryAdapter } = opts.ctx; + const { chainId, address, filters } = opts.input; + return queryAdapter.getRecords({ chainId, address, filters }); + }), + findAll: t.procedure .input( z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a275a971d9..20e7b4c2cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -845,9 +845,6 @@ importers: '@trpc/server': specifier: 10.34.0 version: 10.34.0 - '@wagmi/chains': - specifier: ^0.2.22 - version: 0.2.22(typescript@5.1.6) better-sqlite3: specifier: ^8.6.0 version: 8.6.0 @@ -3681,17 +3678,6 @@ packages: pretty-format: 27.5.1 dev: true - /@wagmi/chains@0.2.22(typescript@5.1.6): - resolution: {integrity: sha512-TdiOzJT6TO1JrztRNjTA5Quz+UmQlbvWFG8N41u9tta0boHA1JCAzGGvU6KuIcOmJfRJkKOUIt67wlbopCpVHg==} - peerDependencies: - typescript: '>=4.9.4' - peerDependenciesMeta: - typescript: - optional: true - dependencies: - typescript: 5.1.6 - dev: false - /abab@2.0.6: resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} dev: true From 0ebc7d390d41b833ccea43e2f9174a239b8b5532 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 16:01:44 +0000 Subject: [PATCH 05/23] missed a spot during rename --- packages/store-sync/src/trpc-indexer/createAppRouter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/store-sync/src/trpc-indexer/createAppRouter.ts b/packages/store-sync/src/trpc-indexer/createAppRouter.ts index 9ed0bd8413..9090c6fa1c 100644 --- a/packages/store-sync/src/trpc-indexer/createAppRouter.ts +++ b/packages/store-sync/src/trpc-indexer/createAppRouter.ts @@ -11,7 +11,7 @@ export function createAppRouter() { }); return t.router({ - getRecords: t.procedure + getLogs: t.procedure .input( z.object({ chainId: z.number(), @@ -27,10 +27,10 @@ export function createAppRouter() { .optional(), }) ) - .query(async (opts): ReturnType => { + .query(async (opts): ReturnType => { const { queryAdapter } = opts.ctx; const { chainId, address, filters } = opts.input; - return queryAdapter.getRecords({ chainId, address, filters }); + return queryAdapter.getLogs({ chainId, address, filters }); }), findAll: t.procedure From d996cd83f1e7fb31b748f856070b81caf122f388 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 16:15:00 +0000 Subject: [PATCH 06/23] reduce log noise --- .../src/postgres/createStorageAdapter.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/store-sync/src/postgres/createStorageAdapter.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts index 9cfbf6deab..89924d3759 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -41,11 +41,15 @@ export async function createStorageAdapter Date: Mon, 27 Nov 2023 17:11:11 +0000 Subject: [PATCH 07/23] replace jsdom with happy-dom --- e2e/packages/sync-test/package.json | 2 +- e2e/packages/sync-test/vite.config.ts | 2 +- e2e/pnpm-lock.yaml | 262 ++------------------------ 3 files changed, 19 insertions(+), 247 deletions(-) diff --git a/e2e/packages/sync-test/package.json b/e2e/packages/sync-test/package.json index 1904d6471f..e6c5fdcc27 100644 --- a/e2e/packages/sync-test/package.json +++ b/e2e/packages/sync-test/package.json @@ -21,7 +21,7 @@ "chalk": "^5.2.0", "dotenv": "^16.0.3", "execa": "^7.1.1", - "jsdom": "^22.0.0", + "happy-dom": "^12.10.3", "typescript": "5.1.6", "viem": "1.14.0", "vite": "^4.2.1", diff --git a/e2e/packages/sync-test/vite.config.ts b/e2e/packages/sync-test/vite.config.ts index c6202f1992..9d916684ff 100644 --- a/e2e/packages/sync-test/vite.config.ts +++ b/e2e/packages/sync-test/vite.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - environment: "jsdom", + environment: "happy-dom", testTimeout: 1000 * 60 * 2, hookTimeout: 1000 * 60 * 2, singleThread: true, diff --git a/e2e/pnpm-lock.yaml b/e2e/pnpm-lock.yaml index 1711ffe418..27aa485a1f 100644 --- a/e2e/pnpm-lock.yaml +++ b/e2e/pnpm-lock.yaml @@ -83,7 +83,7 @@ importers: version: 4.3.5(@types/node@20.1.3) vitest: specifier: 0.31.4 - version: 0.31.4(jsdom@22.0.0) + version: 0.31.4(happy-dom@12.10.3) packages/contracts: devDependencies: @@ -125,7 +125,7 @@ importers: version: 4.3.5(@types/node@20.1.3) vitest: specifier: 0.31.4 - version: 0.31.4(jsdom@22.0.0) + version: 0.31.4(happy-dom@12.10.3) packages/sync-test: devDependencies: @@ -168,9 +168,9 @@ importers: execa: specifier: ^7.1.1 version: 7.1.1 - jsdom: - specifier: ^22.0.0 - version: 22.0.0 + happy-dom: + specifier: ^12.10.3 + version: 12.10.3 typescript: specifier: 5.1.6 version: 5.1.6 @@ -182,7 +182,7 @@ importers: version: 4.3.5(@types/node@20.1.3) vitest: specifier: ^0.31.0 - version: 0.31.4(jsdom@22.0.0) + version: 0.31.4(happy-dom@12.10.3) zod: specifier: ^3.22.2 version: 3.22.2 @@ -754,11 +754,6 @@ packages: '@noble/hashes': 1.3.2 '@scure/base': 1.1.1 - /@tootallnate/once@2.0.0: - resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} - engines: {node: '>= 10'} - dev: true - /@types/chai-subset@1.3.3: resolution: {integrity: sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==} dependencies: @@ -833,10 +828,6 @@ packages: pretty-format: 27.5.1 dev: true - /abab@2.0.6: - resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} - dev: true - /abitype@0.9.8(typescript@5.1.6)(zod@3.22.2): resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} peerDependencies: @@ -866,15 +857,6 @@ packages: hasBin: true dev: true - /agent-base@6.0.2: - resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} - engines: {node: '>= 6.0.0'} - dependencies: - debug: 4.3.4 - transitivePeerDependencies: - - supports-color - dev: true - /ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -901,10 +883,6 @@ packages: tslib: 2.5.0 dev: false - /asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true - /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} dev: true @@ -980,13 +958,6 @@ packages: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} dev: false - /combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: true - /concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} dev: true @@ -1014,20 +985,8 @@ packages: which: 2.0.2 dev: true - /cssstyle@3.0.0: - resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==} - engines: {node: '>=14'} - dependencies: - rrweb-cssom: 0.6.0 - dev: true - - /data-urls@4.0.0: - resolution: {integrity: sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==} - engines: {node: '>=14'} - dependencies: - abab: 2.0.6 - whatwg-mimetype: 3.0.0 - whatwg-url: 12.0.1 + /css.escape@1.5.1: + resolution: {integrity: sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==} dev: true /date-time@3.1.0: @@ -1048,10 +1007,6 @@ packages: dependencies: ms: 2.1.2 - /decimal.js@10.4.3: - resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} - dev: true - /deep-eql@4.1.3: resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} engines: {node: '>=6'} @@ -1059,18 +1014,6 @@ packages: type-detect: 4.0.8 dev: true - /delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: true - - /domexception@4.0.0: - resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==} - engines: {node: '>=12'} - dependencies: - webidl-conversions: 7.0.0 - dev: true - /dotenv@16.0.3: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} engines: {node: '>=12'} @@ -1195,15 +1138,6 @@ packages: optional: true dev: true - /form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: true - /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true @@ -1256,22 +1190,15 @@ packages: resolution: {integrity: sha512-3MSOYFO5U9mPGikIYCzK0SaThypfGgS6bHqrUGXG3DPHCrb+txNqeEcns1W0lkGfk0rCyNXm7xB9rMxnCiZOoA==} dev: false - /html-encoding-sniffer@3.0.0: - resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==} - engines: {node: '>=12'} + /happy-dom@12.10.3: + resolution: {integrity: sha512-JzUXOh0wdNGY54oKng5hliuBkq/+aT1V3YpTM+lrN/GoLQTANZsMaIvmHiHe612rauHvPJnDZkZ+5GZR++1Abg==} dependencies: + css.escape: 1.5.1 + entities: 4.5.0 + iconv-lite: 0.6.3 + webidl-conversions: 7.0.0 whatwg-encoding: 2.0.0 - dev: true - - /http-proxy-agent@5.0.0: - resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} - engines: {node: '>= 6'} - dependencies: - '@tootallnate/once': 2.0.0 - agent-base: 6.0.2 - debug: 4.3.4 - transitivePeerDependencies: - - supports-color + whatwg-mimetype: 3.0.0 dev: true /http-proxy@1.18.1: @@ -1285,16 +1212,6 @@ packages: - debug dev: true - /https-proxy-agent@5.0.1: - resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} - engines: {node: '>= 6'} - dependencies: - agent-base: 6.0.2 - debug: 4.3.4 - transitivePeerDependencies: - - supports-color - dev: true - /human-signals@4.3.1: resolution: {integrity: sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==} engines: {node: '>=14.18.0'} @@ -1328,10 +1245,6 @@ packages: engines: {node: '>=8'} dev: false - /is-potential-custom-element-name@1.0.1: - resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} - dev: true - /is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1361,44 +1274,6 @@ packages: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} dev: false - /jsdom@22.0.0: - resolution: {integrity: sha512-p5ZTEb5h+O+iU02t0GfEjAnkdYPrQSkfuTSMkMYyIoMvUNEHsbG0bHHbfXIcfTqD2UfvjQX7mmgiFsyRwGscVw==} - engines: {node: '>=16'} - peerDependencies: - canvas: ^2.5.0 - peerDependenciesMeta: - canvas: - optional: true - dependencies: - abab: 2.0.6 - cssstyle: 3.0.0 - data-urls: 4.0.0 - decimal.js: 10.4.3 - domexception: 4.0.0 - form-data: 4.0.0 - html-encoding-sniffer: 3.0.0 - http-proxy-agent: 5.0.0 - https-proxy-agent: 5.0.1 - is-potential-custom-element-name: 1.0.1 - nwsapi: 2.2.4 - parse5: 7.1.2 - rrweb-cssom: 0.6.0 - saxes: 6.0.0 - symbol-tree: 3.2.4 - tough-cookie: 4.1.2 - w3c-xmlserializer: 4.0.0 - webidl-conversions: 7.0.0 - whatwg-encoding: 2.0.0 - whatwg-mimetype: 3.0.0 - whatwg-url: 12.0.1 - ws: 8.13.0 - xml-name-validator: 4.0.0 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - dev: true - /jsonc-parser@3.2.0: resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==} dev: true @@ -1461,18 +1336,6 @@ packages: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} dev: true - /mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: true - - /mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - dependencies: - mime-db: 1.52.0 - dev: true - /mimic-fn@4.0.0: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} @@ -1538,10 +1401,6 @@ packages: path-key: 4.0.0 dev: true - /nwsapi@2.2.4: - resolution: {integrity: sha512-NHj4rzRo0tQdijE9ZqAx6kYDcoRwYwSYzCA8MY3JzfxlrvEU0jhnhJT9BhqhJs7I/dKcrDm6TyulaRqZPIhN5g==} - dev: true - /observable-fns@0.6.1: resolution: {integrity: sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg==} dev: false @@ -1566,12 +1425,6 @@ packages: yocto-queue: 1.0.0 dev: true - /parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - dependencies: - entities: 4.5.0 - dev: true - /path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} @@ -1660,19 +1513,6 @@ packages: resolution: {integrity: sha512-kppbvLUNJ4IOMZds9/4gz/rtT5OFiesy3XosLsgMKlF3vb6GA5Y3ptyDlzKLcOcUBW+zaY+RiMINTsgE+O6e+Q==} dev: false - /psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - dev: true - - /punycode@2.3.0: - resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} - engines: {node: '>=6'} - dev: true - - /querystringify@2.2.0: - resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} - dev: true - /react-is@17.0.2: resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} dev: true @@ -1712,10 +1552,6 @@ packages: fsevents: 2.3.2 dev: true - /rrweb-cssom@0.6.0: - resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==} - dev: true - /rxjs@7.5.5: resolution: {integrity: sha512-sy+H0pQofO95VDmFLzyaw9xNJU4KTRSwQIGM6+iG3SypAtCiLDzpeG8sJrNCWn2Up9km+KhkvTdbkrdy+yzZdw==} dependencies: @@ -1726,13 +1562,6 @@ packages: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} dev: true - /saxes@6.0.0: - resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} - engines: {node: '>=v12.22.7'} - dependencies: - xmlchars: 2.2.0 - dev: true - /semver@7.5.0: resolution: {integrity: sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==} engines: {node: '>=10'} @@ -1813,10 +1642,6 @@ packages: acorn: 8.8.2 dev: true - /symbol-tree@3.2.4: - resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} - dev: true - /threads@1.7.0: resolution: {integrity: sha512-Mx5NBSHX3sQYR6iI9VYbgHKBLisyB+xROCBGjjWm1O9wb9vfLxdaGtmT/KCjUqMsSNW6nERzCW3T6H43LqjDZQ==} dependencies: @@ -1857,23 +1682,6 @@ packages: engines: {node: '>=14.0.0'} dev: true - /tough-cookie@4.1.2: - resolution: {integrity: sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==} - engines: {node: '>=6'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - universalify: 0.2.0 - url-parse: 1.5.10 - dev: true - - /tr46@4.1.1: - resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==} - engines: {node: '>=14'} - dependencies: - punycode: 2.3.0 - dev: true - /ts-error@1.0.6: resolution: {integrity: sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==} dev: false @@ -1907,18 +1715,6 @@ packages: resolution: {integrity: sha512-TrY6DsjTQQgyS3E3dBaOXf0TpPD8u9FVrVYmKVegJuFw51n/YB9XPt+U6ydzFG5ZIN7+DIjPbNmXoBj9esYhgQ==} dev: true - /universalify@0.2.0: - resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} - engines: {node: '>= 4.0.0'} - dev: true - - /url-parse@1.5.10: - resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - dev: true - /viem@1.14.0(typescript@5.1.6)(zod@3.22.2): resolution: {integrity: sha512-4d+4/H3lnbkSAbrpQ15i1nBA7hne06joLFy3L3m0ZpMc+g+Zr3D4nuSTyeiqbHAYs9m2P9Kjap0HlyGkehasgg==} peerDependencies: @@ -1996,7 +1792,7 @@ packages: fsevents: 2.3.2 dev: true - /vitest@0.31.4(jsdom@22.0.0): + /vitest@0.31.4(happy-dom@12.10.3): resolution: {integrity: sha512-GoV0VQPmWrUFOZSg3RpQAPN+LPmHg2/gxlMNJlyxJihkz6qReHDV6b0pPDcqFLNEPya4tWJ1pgwUNP9MLmUfvQ==} engines: {node: '>=v14.18.0'} hasBin: true @@ -2041,7 +1837,7 @@ packages: chai: 4.3.7 concordance: 5.0.4 debug: 4.3.4 - jsdom: 22.0.0 + happy-dom: 12.10.3 local-pkg: 0.4.3 magic-string: 0.30.0 pathe: 1.1.0 @@ -2062,13 +1858,6 @@ packages: - terser dev: true - /w3c-xmlserializer@4.0.0: - resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==} - engines: {node: '>=14'} - dependencies: - xml-name-validator: 4.0.0 - dev: true - /webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -2091,14 +1880,6 @@ packages: engines: {node: '>=12'} dev: true - /whatwg-url@12.0.1: - resolution: {integrity: sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==} - engines: {node: '>=14'} - dependencies: - tr46: 4.1.1 - webidl-conversions: 7.0.0 - dev: true - /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -2141,15 +1922,6 @@ packages: utf-8-validate: optional: true - /xml-name-validator@4.0.0: - resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==} - engines: {node: '>=12'} - dev: true - - /xmlchars@2.2.0: - resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} - dev: true - /y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} From b5c4252fbc2fda5d9a7a13e509ae981f6b8af538 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 27 Nov 2023 17:22:38 +0000 Subject: [PATCH 08/23] attempt to fix PID issues --- packages/store-indexer/bin/postgres-indexer.ts | 2 +- packages/store-sync/src/postgres/transformSchemaName.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 946f342376..8812fe5f52 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -7,7 +7,7 @@ import { isDefined } from "@latticexyz/common/utils"; import { combineLatest, filter, first } from "rxjs"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; -import { cleanDatabase, createStorageAdapter, schemaVersion } from "@latticexyz/store-sync/postgres"; +import { createStorageAdapter } from "@latticexyz/store-sync/postgres"; import { createStoreSync } from "@latticexyz/store-sync"; import { indexerEnvSchema, parseEnv } from "./parseEnv"; diff --git a/packages/store-sync/src/postgres/transformSchemaName.ts b/packages/store-sync/src/postgres/transformSchemaName.ts index 353d2c1632..050997a042 100644 --- a/packages/store-sync/src/postgres/transformSchemaName.ts +++ b/packages/store-sync/src/postgres/transformSchemaName.ts @@ -3,7 +3,7 @@ */ export function transformSchemaName(schemaName: string): string { if (process.env.NODE_ENV === "test") { - return `${process.pid}_${process.env.VITEST_POOL_ID}__${schemaName}`; + return `test_${process.env.VITEST_POOL_ID}__${schemaName}`; } return schemaName; } From 7b1c8b1e11e231d7cf06043e59a967523c555890 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 08:57:08 +0000 Subject: [PATCH 09/23] clean up --- packages/common/src/utils/index.ts | 1 + packages/common/src/utils/unique.ts | 3 ++ .../store-indexer/src/postgres/getLogs.ts | 18 +++---- .../store-sync/src/postgres/cleanDatabase.ts | 19 +++++-- .../src/postgres/createStorageAdapter.test.ts | 8 +-- .../src/postgres/createStorageAdapter.ts | 54 ++++++++----------- .../src/postgres/setupTables.test.ts | 14 +++-- .../store-sync/src/postgres/setupTables.ts | 4 +- packages/store-sync/src/postgres/tables.ts | 19 ++++--- 9 files changed, 72 insertions(+), 68 deletions(-) create mode 100644 packages/common/src/utils/unique.ts diff --git a/packages/common/src/utils/index.ts b/packages/common/src/utils/index.ts index c11ae82e51..6bf0ddb0d0 100644 --- a/packages/common/src/utils/index.ts +++ b/packages/common/src/utils/index.ts @@ -9,6 +9,7 @@ export * from "./isDefined"; export * from "./isNotNull"; export * from "./iteratorToArray"; export * from "./mapObject"; +export * from "./unique"; export * from "./uniqueBy"; export * from "./wait"; export * from "./waitForIdle"; diff --git a/packages/common/src/utils/unique.ts b/packages/common/src/utils/unique.ts new file mode 100644 index 0000000000..565cd7b03e --- /dev/null +++ b/packages/common/src/utils/unique.ts @@ -0,0 +1,3 @@ +export function unique(values: readonly value[]): readonly value[] { + return Array.from(new Set(values)); +} diff --git a/packages/store-indexer/src/postgres/getLogs.ts b/packages/store-indexer/src/postgres/getLogs.ts index 4479f77fc3..c39b37aee7 100644 --- a/packages/store-indexer/src/postgres/getLogs.ts +++ b/packages/store-indexer/src/postgres/getLogs.ts @@ -1,7 +1,7 @@ import { PgDatabase } from "drizzle-orm/pg-core"; import { Hex } from "viem"; import { StorageAdapterLog, SyncFilter } from "@latticexyz/store-sync"; -import { chainTable, recordsTable } from "@latticexyz/store-sync/postgres"; +import { tables } from "@latticexyz/store-sync/postgres"; import { and, eq, or } from "drizzle-orm"; import { decodeDynamicField } from "@latticexyz/protocol-parser"; import { bigIntMax } from "@latticexyz/common/utils"; @@ -21,27 +21,27 @@ export async function getLogs( const conditions = filters.length ? filters.map((filter) => and( - address != null ? eq(recordsTable.address, address) : undefined, - eq(recordsTable.tableId, filter.tableId), - filter.key0 != null ? eq(recordsTable.key0, filter.key0) : undefined, - filter.key1 != null ? eq(recordsTable.key1, filter.key1) : undefined + address != null ? eq(tables.recordsTable.address, address) : undefined, + eq(tables.recordsTable.tableId, filter.tableId), + filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined, + filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined ) ) : address != null - ? [eq(recordsTable.address, address)] + ? [eq(tables.recordsTable.address, address)] : []; const chainState = await database .select() - .from(chainTable) - .where(eq(chainTable.chainId, chainId)) + .from(tables.chainTable) + .where(eq(tables.chainTable.chainId, chainId)) .execute() .then((rows) => rows.find(() => true)); let blockNumber = chainState?.lastUpdatedBlockNumber ?? 0n; const records = await database .select() - .from(recordsTable) + .from(tables.recordsTable) .where(or(...conditions)); blockNumber = bigIntMax( blockNumber, diff --git a/packages/store-sync/src/postgres/cleanDatabase.ts b/packages/store-sync/src/postgres/cleanDatabase.ts index f0ccc355d6..e63e897b28 100644 --- a/packages/store-sync/src/postgres/cleanDatabase.ts +++ b/packages/store-sync/src/postgres/cleanDatabase.ts @@ -1,12 +1,23 @@ -import { PgDatabase } from "drizzle-orm/pg-core"; -import { schemaName } from "./tables"; +import { PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; +import { tables } from "./tables"; import { debug } from "./debug"; import { sql } from "drizzle-orm"; import { pgDialect } from "./pgDialect"; +import { isDefined, unique } from "@latticexyz/common/utils"; // This intentionally just cleans up known schemas/tables/rows. We could drop the database but that's scary. export async function cleanDatabase(db: PgDatabase): Promise { - debug(`dropping schema ${schemaName} and all of its tables`); - await db.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); + const schemaNames = unique( + Object.values(tables) + .map((table) => getTableConfig(table).schema) + .filter(isDefined) + ); + + await db.transaction(async (tx) => { + for (const schemaName of schemaNames) { + debug(`dropping schema ${schemaName} and all of its tables`); + await tx.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); + } + }); } diff --git a/packages/store-sync/src/postgres/createStorageAdapter.test.ts b/packages/store-sync/src/postgres/createStorageAdapter.test.ts index 83e02c7f98..a378d1931f 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.test.ts @@ -9,7 +9,7 @@ import { groupLogsByBlockNumber } from "@latticexyz/block-logs-stream"; import { storeEventsAbi } from "@latticexyz/store"; import { StoreEventsLog } from "../common"; import worldRpcLogs from "../../../../test-data/world-logs.json"; -import { chainTable, recordsTable } from "./tables"; +import { tables } from "./tables"; import { resourceToHex } from "@latticexyz/common"; const blocks = groupLogsByBlockNumber( @@ -46,7 +46,7 @@ describe("createStorageAdapter", async () => { await storageAdapter.storageAdapter(block); } - expect(await db.select().from(chainTable)).toMatchInlineSnapshot(` + expect(await db.select().from(tables.chainTable)).toMatchInlineSnapshot(` [ { "chainId": 31337, @@ -58,8 +58,8 @@ describe("createStorageAdapter", async () => { expect( await db .select() - .from(recordsTable) - .where(eq(recordsTable.tableId, resourceToHex({ type: "table", namespace: "", name: "NumberList" }))) + .from(tables.recordsTable) + .where(eq(tables.recordsTable.tableId, resourceToHex({ type: "table", namespace: "", name: "NumberList" }))) ).toMatchInlineSnapshot(` [ { diff --git a/packages/store-sync/src/postgres/createStorageAdapter.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts index 89924d3759..cb9272138b 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -3,17 +3,11 @@ import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; import { and, eq } from "drizzle-orm"; import { StoreConfig } from "@latticexyz/store"; import { debug } from "./debug"; -import { chainTable, storesTable, recordsTable } from "./tables"; +import { tables } from "./tables"; import { spliceHex } from "@latticexyz/common"; import { setupTables } from "./setupTables"; import { StorageAdapter, StorageAdapterBlock } from "../common"; -const tables = { - chainTable, - storesTable, - recordsTable, -} as const; - // Currently assumes one DB per chain ID export type PostgresStorageAdapter = { @@ -38,8 +32,6 @@ export async function createStorageAdapter { await database.transaction(async (tx) => { - // TODO: update stores table - for (const log of logs) { const keyBytes = encodePacked(["bytes32[]"], [log.args.keyTuple]); @@ -51,7 +43,7 @@ export async function createStorageAdapter { describe("before running", () => { it("should be missing schemas", async () => { - await expect(db.select().from(chainTable)).rejects.toThrow(/relation "\w+mud.chain" does not exist/); - await expect(db.select().from(storesTable)).rejects.toThrow(/relation "\w+mud.stores" does not exist/); - await expect(db.select().from(recordsTable)).rejects.toThrow(/relation "\w+mud.records" does not exist/); + await expect(db.select().from(tables.chainTable)).rejects.toThrow(/relation "\w+mud.chain" does not exist/); + await expect(db.select().from(tables.recordsTable)).rejects.toThrow(/relation "\w+mud.records" does not exist/); }); }); describe("after running", () => { beforeEach(async () => { - const cleanUp = await setupTables(db, Object.values([chainTable, storesTable, recordsTable])); + const cleanUp = await setupTables(db, Object.values(tables)); return cleanUp; }); it("should have schemas", async () => { - expect(await db.select().from(chainTable)).toMatchInlineSnapshot("[]"); - expect(await db.select().from(storesTable)).toMatchInlineSnapshot("[]"); - expect(await db.select().from(recordsTable)).toMatchInlineSnapshot("[]"); + expect(await db.select().from(tables.chainTable)).toMatchInlineSnapshot("[]"); + expect(await db.select().from(tables.recordsTable)).toMatchInlineSnapshot("[]"); }); }); }); diff --git a/packages/store-sync/src/postgres/setupTables.ts b/packages/store-sync/src/postgres/setupTables.ts index f3f6c00c79..6c43c170ec 100644 --- a/packages/store-sync/src/postgres/setupTables.ts +++ b/packages/store-sync/src/postgres/setupTables.ts @@ -1,7 +1,7 @@ import { AnyPgColumn, PgTableWithColumns, PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; import { getTableColumns, sql } from "drizzle-orm"; import { ColumnDataType } from "kysely"; -import { isDefined } from "@latticexyz/common/utils"; +import { isDefined, unique } from "@latticexyz/common/utils"; import { debug } from "./debug"; import { pgDialect } from "./pgDialect"; @@ -9,7 +9,7 @@ export async function setupTables( db: PgDatabase, tables: PgTableWithColumns[] ): Promise<() => Promise> { - const schemaNames = [...new Set(tables.map((table) => getTableConfig(table).schema).filter(isDefined))]; + const schemaNames = unique(tables.map((table) => getTableConfig(table).schema).filter(isDefined)); await db.transaction(async (tx) => { for (const schemaName of schemaNames) { diff --git a/packages/store-sync/src/postgres/tables.ts b/packages/store-sync/src/postgres/tables.ts index 06a590b2a1..40ec19a7ca 100644 --- a/packages/store-sync/src/postgres/tables.ts +++ b/packages/store-sync/src/postgres/tables.ts @@ -1,24 +1,18 @@ -import { boolean, index, pgSchema, primaryKey, text } from "drizzle-orm/pg-core"; +import { boolean, index, pgSchema, primaryKey } from "drizzle-orm/pg-core"; import { transformSchemaName } from "./transformSchemaName"; import { asAddress, asBigInt, asHex, asNumber } from "./columnTypes"; -export const schemaName = transformSchemaName("mud"); +const schemaName = transformSchemaName("mud"); /** * Singleton table for the state of the chain we're indexing */ -export const chainTable = pgSchema(schemaName).table("chain", { +const chainTable = pgSchema(schemaName).table("chain", { chainId: asNumber("chain_id", "bigint").notNull().primaryKey(), lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), }); -export const storesTable = pgSchema(schemaName).table("stores", { - address: asAddress("address").notNull().primaryKey(), - storeVersion: text("store_version").notNull(), - lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), -}); - -export const recordsTable = pgSchema(schemaName).table( +const recordsTable = pgSchema(schemaName).table( "records", { address: asAddress("address").notNull(), @@ -43,3 +37,8 @@ export const recordsTable = pgSchema(schemaName).table( // TODO: add indices for querying multiple keys }) ); + +export const tables = { + chainTable, + recordsTable, +}; From d3381b8f7e2e50411f0c2a95088639d8e1551e30 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 10:59:55 +0000 Subject: [PATCH 10/23] simplify postgres decoded using postgres adapter --- .../src/postgres/createQueryAdapter.ts | 13 +- .../src/postgres-decoded/buildColumn.ts | 2 +- .../postgres-decoded/buildInternalTables.ts | 32 - .../src/postgres-decoded/buildTable.test.ts | 592 +----------------- .../src/postgres-decoded/buildTable.ts | 16 +- .../src/postgres-decoded/cleanDatabase.ts | 21 +- .../src/postgres-decoded/columnTypes.ts | 74 --- ...e.test.ts => createStorageAdapter.test.ts} | 57 +- .../postgres-decoded/createStorageAdapter.ts | 145 +++++ .../store-sync/src/postgres-decoded/debug.ts | 2 +- .../src/postgres-decoded/getTableKey.ts | 8 - .../src/postgres-decoded/getTables.ts | 57 +- .../store-sync/src/postgres-decoded/index.ts | 6 +- .../src/postgres-decoded/pgDialect.ts | 10 - .../src/postgres-decoded/postgresStorage.ts | 294 --------- .../src/postgres-decoded/schemaVersion.ts | 4 - .../src/postgres-decoded/setupTables.test.ts | 46 -- .../src/postgres-decoded/setupTables.ts | 62 -- .../src/postgres-decoded/syncToPostgres.ts | 2 +- .../postgres-decoded/transformSchemaName.ts | 4 - .../src/postgres/createStorageAdapter.test.ts | 12 +- packages/store-sync/src/postgres/index.ts | 1 - .../store-sync/src/postgres/schemaVersion.ts | 4 - .../store-sync/src/postgres/setupTables.ts | 7 +- 24 files changed, 267 insertions(+), 1204 deletions(-) delete mode 100644 packages/store-sync/src/postgres-decoded/buildInternalTables.ts delete mode 100644 packages/store-sync/src/postgres-decoded/columnTypes.ts rename packages/store-sync/src/postgres-decoded/{postgresStorage.test.ts => createStorageAdapter.test.ts} (63%) create mode 100644 packages/store-sync/src/postgres-decoded/createStorageAdapter.ts delete mode 100644 packages/store-sync/src/postgres-decoded/getTableKey.ts delete mode 100644 packages/store-sync/src/postgres-decoded/pgDialect.ts delete mode 100644 packages/store-sync/src/postgres-decoded/postgresStorage.ts delete mode 100644 packages/store-sync/src/postgres-decoded/schemaVersion.ts delete mode 100644 packages/store-sync/src/postgres-decoded/setupTables.test.ts delete mode 100644 packages/store-sync/src/postgres-decoded/setupTables.ts delete mode 100644 packages/store-sync/src/postgres-decoded/transformSchemaName.ts delete mode 100644 packages/store-sync/src/postgres/schemaVersion.ts diff --git a/packages/store-indexer/src/postgres/createQueryAdapter.ts b/packages/store-indexer/src/postgres/createQueryAdapter.ts index 1e60f26f65..6f2f3b1cbb 100644 --- a/packages/store-indexer/src/postgres/createQueryAdapter.ts +++ b/packages/store-indexer/src/postgres/createQueryAdapter.ts @@ -1,10 +1,10 @@ +import { getAddress } from "viem"; import { PgDatabase } from "drizzle-orm/pg-core"; +import { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from "@latticexyz/store-sync"; +import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer"; import { debug } from "../debug"; -import { getAddress } from "viem"; -import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; import { getLogs } from "./getLogs"; -import { TableWithRecords, isTableRegistrationLog, logToTable } from "@latticexyz/store-sync"; /** * Creates a query adapter for the tRPC server/client to query data from Postgres. @@ -18,7 +18,12 @@ export async function createQueryAdapter(database: PgDatabase): Promise 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [], + }); const tables = logs.filter(isTableRegistrationLog).map(logToTable); diff --git a/packages/store-sync/src/postgres-decoded/buildColumn.ts b/packages/store-sync/src/postgres-decoded/buildColumn.ts index d9ffa6db99..71e57bd770 100644 --- a/packages/store-sync/src/postgres-decoded/buildColumn.ts +++ b/packages/store-sync/src/postgres-decoded/buildColumn.ts @@ -1,7 +1,7 @@ import { boolean, text } from "drizzle-orm/pg-core"; import { SchemaAbiType } from "@latticexyz/schema-type"; import { assertExhaustive } from "@latticexyz/common/utils"; -import { asAddress, asBigInt, asHex, asJson, asNumber } from "./columnTypes"; +import { asAddress, asBigInt, asHex, asJson, asNumber } from "../postgres/columnTypes"; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function buildColumn(name: string, schemaAbiType: SchemaAbiType) { diff --git a/packages/store-sync/src/postgres-decoded/buildInternalTables.ts b/packages/store-sync/src/postgres-decoded/buildInternalTables.ts deleted file mode 100644 index bd0b7372e0..0000000000 --- a/packages/store-sync/src/postgres-decoded/buildInternalTables.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { integer, pgSchema, text } from "drizzle-orm/pg-core"; -import { transformSchemaName } from "./transformSchemaName"; -import { asAddress, asBigInt, asHex, asJson, asNumber } from "./columnTypes"; -import { KeySchema, ValueSchema } from "@latticexyz/protocol-parser"; - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function buildInternalTables() { - const schema = pgSchema(transformSchemaName("__mud_internal")); - return { - chain: schema.table("chain", { - // TODO: change schema version to varchar/text? - schemaVersion: integer("schema_version").notNull().primaryKey(), - chainId: asNumber("chain_id", "bigint").notNull().primaryKey(), - lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), - // TODO: last block hash? - lastError: text("last_error"), - }), - tables: schema.table("tables", { - schemaVersion: integer("schema_version").primaryKey(), - key: text("key").notNull().primaryKey(), - address: asAddress("address").notNull(), - tableId: asHex("table_id").notNull(), - namespace: text("namespace").notNull(), - name: text("name").notNull(), - keySchema: asJson("key_schema").notNull(), - valueSchema: asJson("value_schema").notNull(), - lastUpdatedBlockNumber: asBigInt("last_updated_block_number", "numeric"), - // TODO: last block hash? - lastError: text("last_error"), - }), - }; -} diff --git a/packages/store-sync/src/postgres-decoded/buildTable.test.ts b/packages/store-sync/src/postgres-decoded/buildTable.test.ts index b13b991393..bddf3ff451 100644 --- a/packages/store-sync/src/postgres-decoded/buildTable.test.ts +++ b/packages/store-sync/src/postgres-decoded/buildTable.test.ts @@ -13,106 +13,7 @@ describe("buildTable", () => { expect(table).toMatchInlineSnapshot(` PgTable { - "__dynamicData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__dynamicData", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__dynamicData", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "__encodedLengths": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__encodedLengths", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__encodedLengths", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "__isDeleted": PgBoolean { - "columnType": "PgBoolean", - "config": { - "columnType": "PgBoolean", - "dataType": "boolean", - "default": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "dataType": "boolean", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "__key": PgCustomColumn { + "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", "config": { "columnType": "PgCustomColumn", @@ -162,46 +63,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "__staticData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primaryKey": false, - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -212,12 +77,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primary": false, - "sqlName": "bytea", + "sqlName": "numeric", "table": [Circular], - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "addr": PgCustomColumn { @@ -358,108 +223,9 @@ describe("buildTable", () => { }, Symbol(drizzle:Name): "users", Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", + Symbol(drizzle:Schema): "test_4__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", Symbol(drizzle:Columns): { - "__dynamicData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__dynamicData", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__dynamicData", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "__encodedLengths": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__encodedLengths", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__encodedLengths", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "__isDeleted": PgBoolean { - "columnType": "PgBoolean", - "config": { - "columnType": "PgBoolean", - "dataType": "boolean", - "default": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "dataType": "boolean", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "__key": PgCustomColumn { + "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", "config": { "columnType": "PgCustomColumn", @@ -509,46 +275,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "__staticData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primaryKey": false, - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -559,12 +289,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primary": false, - "sqlName": "bytea", + "sqlName": "numeric", "table": [Circular], - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "addr": PgCustomColumn { @@ -724,106 +454,7 @@ describe("buildTable", () => { expect(table).toMatchInlineSnapshot(` PgTable { - "__dynamicData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__dynamicData", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__dynamicData", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "__encodedLengths": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__encodedLengths", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__encodedLengths", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "__isDeleted": PgBoolean { - "columnType": "PgBoolean", - "config": { - "columnType": "PgBoolean", - "dataType": "boolean", - "default": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "dataType": "boolean", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "__key": PgCustomColumn { + "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", "config": { "columnType": "PgCustomColumn", @@ -873,46 +504,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "__staticData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primaryKey": false, - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -923,12 +518,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primary": false, - "sqlName": "bytea", + "sqlName": "numeric", "table": [Circular], - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "addrs": PgCustomColumn { @@ -969,108 +564,9 @@ describe("buildTable", () => { }, Symbol(drizzle:Name): "users", Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", + Symbol(drizzle:Schema): "test_4__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", Symbol(drizzle:Columns): { - "__dynamicData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__dynamicData", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__dynamicData", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___dynamicData_unique", - "uniqueType": undefined, - }, - "__encodedLengths": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__encodedLengths", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__encodedLengths", - "notNull": false, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___encodedLengths_unique", - "uniqueType": undefined, - }, - "__isDeleted": PgBoolean { - "columnType": "PgBoolean", - "config": { - "columnType": "PgBoolean", - "dataType": "boolean", - "default": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "dataType": "boolean", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__isDeleted", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users___isDeleted_unique", - "uniqueType": undefined, - }, - "__key": PgCustomColumn { + "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", "config": { "columnType": "PgCustomColumn", @@ -1120,46 +616,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primaryKey": false, - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__lastUpdatedBlockNumber", - "notNull": true, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___lastUpdatedBlockNumber_unique", - "uniqueType": undefined, - }, - "__staticData": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primaryKey": false, - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -1170,12 +630,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__staticData", + "name": "__last_updated_block_number", "notNull": false, "primary": false, - "sqlName": "bytea", + "sqlName": "numeric", "table": [Circular], - "uniqueName": "users___staticData_unique", + "uniqueName": "users___last_updated_block_number_unique", "uniqueType": undefined, }, "addrs": PgCustomColumn { diff --git a/packages/store-sync/src/postgres-decoded/buildTable.ts b/packages/store-sync/src/postgres-decoded/buildTable.ts index 2fb7feb499..131f5c3d58 100644 --- a/packages/store-sync/src/postgres-decoded/buildTable.ts +++ b/packages/store-sync/src/postgres-decoded/buildTable.ts @@ -1,18 +1,13 @@ import { PgColumnBuilderBase, PgTableWithColumns, pgSchema } from "drizzle-orm/pg-core"; -import { buildColumn } from "./buildColumn"; import { Address, getAddress } from "viem"; -import { transformSchemaName } from "./transformSchemaName"; import { KeySchema, ValueSchema } from "@latticexyz/protocol-parser"; +import { asBigInt, asHex } from "../postgres/columnTypes"; +import { transformSchemaName } from "../postgres/transformSchemaName"; +import { buildColumn } from "./buildColumn"; -// TODO: convert camel case to snake case for DB storage? export const metaColumns = { - __key: buildColumn("__key", "bytes").primaryKey(), - __staticData: buildColumn("__staticData", "bytes"), - __encodedLengths: buildColumn("__encodedLengths", "bytes"), - __dynamicData: buildColumn("__dynamicData", "bytes"), - __lastUpdatedBlockNumber: buildColumn("__lastUpdatedBlockNumber", "uint256").notNull(), - // TODO: last updated block hash? - __isDeleted: buildColumn("__isDeleted", "bool").notNull(), + __keyBytes: asHex("__key_bytes").primaryKey(), + __lastUpdatedBlockNumber: asBigInt("__last_updated_block_number", "numeric"), } as const satisfies Record; type PgTableFromSchema = PgTableWithColumns<{ @@ -62,7 +57,6 @@ export function buildTable): Promise { - const internalTables = buildInternalTables(); - // TODO: check if internalTables schema matches, delete if not + const sqlTables = (await getTables(db)).map(buildTable); - const tables = (await getTables(db)).map(buildTable); - - const schemaNames = [...new Set(tables.map((table) => getTableConfig(table).schema))].filter(isDefined); + const schemaNames = unique(sqlTables.map((sqlTable) => getTableConfig(sqlTable).schema).filter(isDefined)); for (const schemaName of schemaNames) { try { @@ -26,9 +23,5 @@ export async function cleanDatabase(db: PgDatabase): Promise { } } - for (const internalTable of Object.values(internalTables)) { - const tableConfig = getTableConfig(internalTable); - debug(`deleting all rows from ${tableConfig.schema}.${tableConfig.name}`); - await db.delete(internalTable); - } + await cleanBytesDatabase(db); } diff --git a/packages/store-sync/src/postgres-decoded/columnTypes.ts b/packages/store-sync/src/postgres-decoded/columnTypes.ts deleted file mode 100644 index da9d9761f8..0000000000 --- a/packages/store-sync/src/postgres-decoded/columnTypes.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { customType } from "drizzle-orm/pg-core"; -import superjson from "superjson"; -import { Address, ByteArray, bytesToHex, getAddress, Hex, hexToBytes } from "viem"; - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const asJson = (name: string) => - customType<{ data: TData; driverData: string }>({ - dataType() { - // TODO: move to json column type? if we do, we'll prob wanna choose something other than superjson since it adds one level of depth (json/meta keys) - return "text"; - }, - toDriver(data: TData): string { - return superjson.stringify(data); - }, - fromDriver(driverData: string): TData { - return superjson.parse(driverData); - }, - })(name); - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const asNumber = (name: string, columnType: string) => - customType<{ data: number; driverData: string }>({ - dataType() { - return columnType; - }, - toDriver(data: number): string { - return String(data); - }, - fromDriver(driverData: string): number { - return Number(driverData); - }, - })(name); - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const asBigInt = (name: string, columnType: string) => - customType<{ data: bigint; driverData: string }>({ - dataType() { - return columnType; - }, - toDriver(data: bigint): string { - return String(data); - }, - fromDriver(driverData: string): bigint { - return BigInt(driverData); - }, - })(name); - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const asHex = (name: string) => - customType<{ data: Hex; driverData: ByteArray }>({ - dataType() { - return "bytea"; - }, - toDriver(data: Hex): ByteArray { - return hexToBytes(data); - }, - fromDriver(driverData: ByteArray): Hex { - return bytesToHex(driverData); - }, - })(name); - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const asAddress = (name: string) => - customType<{ data: Address; driverData: ByteArray }>({ - dataType() { - return "bytea"; - }, - toDriver(data: Address): ByteArray { - return hexToBytes(data); - }, - fromDriver(driverData: ByteArray): Address { - return getAddress(bytesToHex(driverData)); - }, - })(name); diff --git a/packages/store-sync/src/postgres-decoded/postgresStorage.test.ts b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts similarity index 63% rename from packages/store-sync/src/postgres-decoded/postgresStorage.test.ts rename to packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts index 846fcf3f65..a0e67060aa 100644 --- a/packages/store-sync/src/postgres-decoded/postgresStorage.test.ts +++ b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts @@ -1,17 +1,17 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it } from "vitest"; import { DefaultLogger, eq } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { Hex, RpcLog, createPublicClient, decodeEventLog, formatLog, http } from "viem"; import { foundry } from "viem/chains"; -import * as transformSchemaNameExports from "./transformSchemaName"; import { getTables } from "./getTables"; -import { PostgresStorageAdapter, postgresStorage } from "./postgresStorage"; +import { PostgresStorageAdapter, createStorageAdapter } from "./createStorageAdapter"; import { buildTable } from "./buildTable"; import { groupLogsByBlockNumber } from "@latticexyz/block-logs-stream"; import { storeEventsAbi } from "@latticexyz/store"; import { StoreEventsLog } from "../common"; import worldRpcLogs from "../../../../test-data/world-logs.json"; +import { resourceToHex } from "@latticexyz/common"; const blocks = groupLogsByBlockNumber( worldRpcLogs.map((log) => { @@ -25,11 +25,7 @@ const blocks = groupLogsByBlockNumber( }) ); -vi.spyOn(transformSchemaNameExports, "transformSchemaName").mockImplementation( - (schemaName) => `${process.pid}_${process.env.VITEST_POOL_ID}__${schemaName}` -); - -describe("postgresStorage", async () => { +describe("createStorageAdapter", async () => { const db = drizzle(postgres(process.env.DATABASE_URL!), { logger: new DefaultLogger(), }); @@ -42,7 +38,7 @@ describe("postgresStorage", async () => { let storageAdapter: PostgresStorageAdapter; beforeEach(async () => { - storageAdapter = await postgresStorage({ database: db, publicClient }); + storageAdapter = await createStorageAdapter({ database: db, publicClient }); return storageAdapter.cleanUp; }); @@ -51,13 +47,11 @@ describe("postgresStorage", async () => { await storageAdapter.storageAdapter(block); } - expect(await db.select().from(storageAdapter.internalTables.chain)).toMatchInlineSnapshot(` + expect(await db.select().from(storageAdapter.tables.chainTable)).toMatchInlineSnapshot(` [ { "chainId": 31337, - "lastError": null, "lastUpdatedBlockNumber": 12n, - "schemaVersion": 1, }, ] `); @@ -65,23 +59,26 @@ describe("postgresStorage", async () => { expect( await db .select() - .from(storageAdapter.internalTables.tables) - .where(eq(storageAdapter.internalTables.tables.name, "NumberList")) + .from(storageAdapter.tables.recordsTable) + .where( + eq( + storageAdapter.tables.recordsTable.tableId, + resourceToHex({ type: "table", namespace: "", name: "NumberList" }) + ) + ) ).toMatchInlineSnapshot(` [ { "address": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342", - "key": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342::NumberList", - "keySchema": {}, - "lastError": null, + "dynamicData": "0x000001a400000045", + "encodedLengths": "0x0000000000000000000000000000000000000000000000000800000000000008", + "isDeleted": false, + "key0": null, + "key1": null, + "keyBytes": "0x", "lastUpdatedBlockNumber": 12n, - "name": "NumberList", - "namespace": "", - "schemaVersion": 1, + "staticData": null, "tableId": "0x746200000000000000000000000000004e756d6265724c697374000000000000", - "valueSchema": { - "value": "uint32[]", - }, }, ] `); @@ -91,13 +88,9 @@ describe("postgresStorage", async () => { [ { "address": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342", - "key": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342::NumberList", "keySchema": {}, - "lastError": null, - "lastUpdatedBlockNumber": 12n, "name": "NumberList", "namespace": "", - "schemaVersion": 1, "tableId": "0x746200000000000000000000000000004e756d6265724c697374000000000000", "valueSchema": { "value": "uint32[]", @@ -110,15 +103,11 @@ describe("postgresStorage", async () => { expect(await db.select().from(sqlTable)).toMatchInlineSnapshot(` [ { - "__dynamicData": "0x000001a400000045", - "__encodedLengths": "0x0000000000000000000000000000000000000000000000000800000000000008", - "__isDeleted": false, - "__key": "0x", + "__keyBytes": "0x", "__lastUpdatedBlockNumber": 12n, - "__staticData": null, "value": [ - 420, - 69, + 0, + 0, ], }, ] diff --git a/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts b/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts new file mode 100644 index 0000000000..b8e84de9eb --- /dev/null +++ b/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts @@ -0,0 +1,145 @@ +import { Hex, PublicClient, concatHex, getAddress } from "viem"; +import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; +import { and, eq } from "drizzle-orm"; +import { buildTable } from "./buildTable"; +import { StoreConfig } from "@latticexyz/store"; +import { debug } from "./debug"; +import { StorageAdapter, StorageAdapterBlock } from "../common"; +import { isTableRegistrationLog } from "../isTableRegistrationLog"; +import { logToTable } from "../logToTable"; +import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; +import { tables as internalTables } from "../postgres/tables"; +import { createStorageAdapter as createBytesStorageAdapter } from "../postgres/createStorageAdapter"; +import { setupTables } from "../postgres/setupTables"; +import { getTables } from "./getTables"; +import { hexToResource } from "@latticexyz/common"; + +// Currently assumes one DB per chain ID + +export type PostgresStorageAdapter = { + storageAdapter: StorageAdapter; + tables: typeof internalTables; + cleanUp: () => Promise; +}; + +export async function createStorageAdapter({ + database, + publicClient, + config, +}: { + database: PgDatabase; + publicClient: PublicClient; + config?: TConfig; +}): Promise { + const bytesStorageAdapter = await createBytesStorageAdapter({ database, publicClient, config }); + const cleanUp: (() => Promise)[] = []; + + async function postgresStorageAdapter({ blockNumber, logs }: StorageAdapterBlock): Promise { + await bytesStorageAdapter.storageAdapter({ blockNumber, logs }); + + const newTables = logs.filter(isTableRegistrationLog).map(logToTable); + const newSqlTables = newTables.map(buildTable); + cleanUp.push(await setupTables(database, newSqlTables)); + + // TODO: cache these inside `getTables`? + const tables = await getTables( + database, + logs.map((log) => ({ address: log.address, tableId: log.args.tableId })) + ); + + // TODO: check if DB schema/table was created? + + // This is currently parallelized per world (each world has its own database). + // This may need to change if we decide to put multiple worlds into one DB (e.g. a namespace per world, but all under one DB). + // If so, we'll probably want to wrap the entire block worth of operations in a transaction. + + await database.transaction(async (tx) => { + for (const log of logs) { + const table = tables.find( + (table) => getAddress(table.address) === getAddress(log.address) && table.tableId === log.args.tableId + ); + if (!table) { + const { namespace, name } = hexToResource(log.args.tableId); + debug(`table registration record for ${namespace}:${name} not found, skipping log`, log); + continue; + } + + const sqlTable = buildTable(table); + const keyBytes = concatHex(log.args.keyTuple as Hex[]); + const key = decodeKey(table.keySchema, log.args.keyTuple); + + if ( + log.eventName === "Store_SetRecord" || + log.eventName === "Store_SpliceStaticData" || + log.eventName === "Store_SpliceDynamicData" + ) { + const record = await database + .select() + .from(internalTables.recordsTable) + .where( + and( + eq(internalTables.recordsTable.address, log.address), + eq(internalTables.recordsTable.tableId, log.args.tableId), + eq(internalTables.recordsTable.keyBytes, keyBytes) + ) + ) + .limit(1) + .then((rows) => rows.find(() => true)); + if (!record) { + const { namespace, name } = hexToResource(log.args.tableId); + debug(`no record found for ${log.args.keyTuple} in table ${namespace}:${name}, skipping log`, log); + continue; + } + + const value = decodeValueArgs(table.valueSchema, { + staticData: record.staticData ?? "0x", + encodedLengths: record.encodedLengths ?? "0x", + dynamicData: record.encodedLengths ?? "0x", + }); + + debug("upserting record", { + namespace: table.namespace, + name: table.name, + key, + }); + + await tx + .insert(sqlTable) + .values({ + __keyBytes: keyBytes, + __lastUpdatedBlockNumber: blockNumber, + ...key, + ...value, + }) + .onConflictDoUpdate({ + target: sqlTable.__keyBytes, + set: { + __lastUpdatedBlockNumber: blockNumber, + ...value, + }, + }) + .execute(); + } else if (log.eventName === "Store_DeleteRecord") { + debug("deleting record", { + namespace: table.namespace, + name: table.name, + key, + }); + + await tx.delete(sqlTable).where(eq(sqlTable.__keyBytes, keyBytes)).execute(); + } + } + }); + } + + return { + storageAdapter: postgresStorageAdapter, + tables: internalTables, + cleanUp: async (): Promise => { + for (const fn of cleanUp) { + await fn(); + } + await bytesStorageAdapter.cleanUp(); + }, + }; +} diff --git a/packages/store-sync/src/postgres-decoded/debug.ts b/packages/store-sync/src/postgres-decoded/debug.ts index 306f33b44b..607afe7505 100644 --- a/packages/store-sync/src/postgres-decoded/debug.ts +++ b/packages/store-sync/src/postgres-decoded/debug.ts @@ -1,3 +1,3 @@ import { debug as parentDebug } from "../debug"; -export const debug = parentDebug.extend("postgres"); +export const debug = parentDebug.extend("postgres-decoded"); diff --git a/packages/store-sync/src/postgres-decoded/getTableKey.ts b/packages/store-sync/src/postgres-decoded/getTableKey.ts deleted file mode 100644 index c1d08a0c29..0000000000 --- a/packages/store-sync/src/postgres-decoded/getTableKey.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { getAddress } from "viem"; -import { Table } from "../common"; -import { hexToResource } from "@latticexyz/common"; - -export function getTableKey({ address, tableId }: Pick): string { - const { namespace, name } = hexToResource(tableId); - return `${getAddress(address)}:${namespace}:${name}`; -} diff --git a/packages/store-sync/src/postgres-decoded/getTables.ts b/packages/store-sync/src/postgres-decoded/getTables.ts index b218ec2f03..b3ba2bd1ea 100644 --- a/packages/store-sync/src/postgres-decoded/getTables.ts +++ b/packages/store-sync/src/postgres-decoded/getTables.ts @@ -1,30 +1,43 @@ import { PgDatabase } from "drizzle-orm/pg-core"; -import { getTableName, inArray } from "drizzle-orm"; -import { Table } from "../common"; -import { buildInternalTables } from "./buildInternalTables"; -import { buildTable } from "./buildTable"; -import { debug } from "./debug"; -import { isDefined } from "@latticexyz/common/utils"; +import { and, eq, or } from "drizzle-orm"; +import { Table, storeTables } from "../common"; +import { tables as internalTables } from "../postgres/tables"; +import { Hex } from "viem"; +import { decodeDynamicField } from "@latticexyz/protocol-parser"; +import { logToTable } from "../logToTable"; -export async function getTables(db: PgDatabase, keys: string[] = []): Promise { - const internalTables = buildInternalTables(); +export async function getTables( + db: PgDatabase, + filters: { address: Hex | null; tableId: Hex | null }[] = [] +): Promise { + const conditions = filters.map((filter) => + and( + filter.address != null ? eq(internalTables.recordsTable.address, filter.address) : undefined, + filter.tableId != null ? eq(internalTables.recordsTable.key0, filter.tableId) : undefined + ) + ); - const tables = await db + const records = await db .select() - .from(internalTables.tables) - .where(keys.length ? inArray(internalTables.tables.key, [...new Set(keys)]) : undefined); + .from(internalTables.recordsTable) + .where(and(eq(internalTables.recordsTable.tableId, storeTables.Tables.tableId), or(...conditions))); - const validTables = await Promise.all( - tables.map(async (table) => { - const sqlTable = buildTable(table); - try { - await db.select({ key: sqlTable.__key }).from(sqlTable).limit(1); - return table; - } catch (error) { - debug("Could not query table, skipping", getTableName(sqlTable), error); - } - }) + const logs = records.map( + (record) => + ({ + address: record.address, + eventName: "Store_SetRecord", + args: { + tableId: record.tableId, + keyTuple: decodeDynamicField("bytes32[]", record.keyBytes), + staticData: record.staticData ?? "0x", + encodedLengths: record.encodedLengths ?? "0x", + dynamicData: record.dynamicData ?? "0x", + }, + } as const) ); - return validTables.filter(isDefined); + const tables = logs.map(logToTable); + + return tables; } diff --git a/packages/store-sync/src/postgres-decoded/index.ts b/packages/store-sync/src/postgres-decoded/index.ts index dd1f7c115f..76ed5fcfb7 100644 --- a/packages/store-sync/src/postgres-decoded/index.ts +++ b/packages/store-sync/src/postgres-decoded/index.ts @@ -1,9 +1,5 @@ export * from "./buildTable"; export * from "./cleanDatabase"; +export * from "./createStorageAdapter"; export * from "./getTables"; -export * from "./buildInternalTables"; -export * from "./schemaVersion"; -export * from "./postgresStorage"; -export * from "./setupTables"; export * from "./syncToPostgres"; -export * from "./columnTypes"; diff --git a/packages/store-sync/src/postgres-decoded/pgDialect.ts b/packages/store-sync/src/postgres-decoded/pgDialect.ts deleted file mode 100644 index f5f42a526b..0000000000 --- a/packages/store-sync/src/postgres-decoded/pgDialect.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { DummyDriver, Kysely, PostgresAdapter, PostgresIntrospector, PostgresQueryCompiler } from "kysely"; - -export const pgDialect = new Kysely({ - dialect: { - createAdapter: (): PostgresAdapter => new PostgresAdapter(), - createDriver: (): DummyDriver => new DummyDriver(), - createIntrospector: (db: Kysely): PostgresIntrospector => new PostgresIntrospector(db), - createQueryCompiler: (): PostgresQueryCompiler => new PostgresQueryCompiler(), - }, -}); diff --git a/packages/store-sync/src/postgres-decoded/postgresStorage.ts b/packages/store-sync/src/postgres-decoded/postgresStorage.ts deleted file mode 100644 index 111894aada..0000000000 --- a/packages/store-sync/src/postgres-decoded/postgresStorage.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { Hex, PublicClient, concatHex, size } from "viem"; -import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; -import { eq, getTableName, inArray } from "drizzle-orm"; -import { buildTable } from "./buildTable"; -import { StoreConfig } from "@latticexyz/store"; -import { debug } from "./debug"; -import { buildInternalTables } from "./buildInternalTables"; -import { getTables } from "./getTables"; -import { schemaVersion } from "./schemaVersion"; -import { hexToResource, spliceHex } from "@latticexyz/common"; -import { setupTables } from "./setupTables"; -import { getTableKey } from "./getTableKey"; -import { StorageAdapter, StorageAdapterBlock } from "../common"; -import { isTableRegistrationLog } from "../isTableRegistrationLog"; -import { logToTable } from "../logToTable"; -import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; - -// Currently assumes one DB per chain ID - -export type PostgresStorageAdapter = { - storageAdapter: StorageAdapter; - internalTables: ReturnType; - cleanUp: () => Promise; -}; - -export async function postgresStorage({ - database, - publicClient, -}: { - database: PgDatabase; - publicClient: PublicClient; - config?: TConfig; -}): Promise { - const cleanUp: (() => Promise)[] = []; - - const chainId = publicClient.chain?.id ?? (await publicClient.getChainId()); - - const internalTables = buildInternalTables(); - cleanUp.push(await setupTables(database, Object.values(internalTables))); - - async function postgresStorageAdapter({ blockNumber, logs }: StorageAdapterBlock): Promise { - const newTables = logs.filter(isTableRegistrationLog).map(logToTable); - const newSqlTables = newTables.map(buildTable); - - cleanUp.push(await setupTables(database, newSqlTables)); - - await database.transaction(async (tx) => { - for (const table of newTables) { - await tx - .insert(internalTables.tables) - .values({ - schemaVersion, - key: getTableKey(table), - ...table, - lastUpdatedBlockNumber: blockNumber, - }) - .onConflictDoNothing() - .execute(); - } - }); - - const tables = await getTables( - database, - logs.map((log) => getTableKey({ address: log.address, tableId: log.args.tableId })) - ); - - // This is currently parallelized per world (each world has its own database). - // This may need to change if we decide to put multiple worlds into one DB (e.g. a namespace per world, but all under one DB). - // If so, we'll probably want to wrap the entire block worth of operations in a transaction. - - await database.transaction(async (tx) => { - const tablesWithOperations = tables.filter((table) => - logs.some((log) => getTableKey({ address: log.address, tableId: log.args.tableId }) === getTableKey(table)) - ); - if (tablesWithOperations.length) { - await tx - .update(internalTables.tables) - .set({ lastUpdatedBlockNumber: blockNumber }) - .where(inArray(internalTables.tables.key, [...new Set(tablesWithOperations.map(getTableKey))])) - .execute(); - } - - for (const log of logs) { - const table = tables.find( - (table) => getTableKey(table) === getTableKey({ address: log.address, tableId: log.args.tableId }) - ); - if (!table) { - const { namespace, name } = hexToResource(log.args.tableId); - debug(`table ${namespace}:${name} not found, skipping log`, log); - continue; - } - - const sqlTable = buildTable(table); - const uniqueKey = concatHex(log.args.keyTuple as Hex[]); - const key = decodeKey(table.keySchema, log.args.keyTuple); - - debug(log.eventName, log); - - if (log.eventName === "Store_SetRecord") { - const value = decodeValueArgs(table.valueSchema, log.args); - debug("upserting record", { - namespace: table.namespace, - name: table.name, - key, - value, - }); - await tx - .insert(sqlTable) - .values({ - __key: uniqueKey, - __staticData: log.args.staticData, - __encodedLengths: log.args.encodedLengths, - __dynamicData: log.args.dynamicData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...key, - ...value, - }) - .onConflictDoUpdate({ - target: sqlTable.__key, - set: { - __staticData: log.args.staticData, - __encodedLengths: log.args.encodedLengths, - __dynamicData: log.args.dynamicData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...value, - }, - }) - .execute(); - } else if (log.eventName === "Store_SpliceStaticData") { - // TODO: verify that this returns what we expect (doesn't error/undefined on no record) - const previousValue = await tx - .select() - .from(sqlTable) - .where(eq(sqlTable.__key, uniqueKey)) - .execute() - .then( - (rows) => rows[0], - (error) => (error instanceof Error ? error : new Error(String(error))) - ); - if (previousValue instanceof Error) { - // https://github.com/latticexyz/mud/issues/1923 - debug( - "Could not query previous value for splice static data, skipping update", - getTableName(sqlTable), - uniqueKey, - previousValue - ); - continue; - } - const previousStaticData = (previousValue?.__staticData as Hex) ?? "0x"; - const newStaticData = spliceHex(previousStaticData, log.args.start, size(log.args.data), log.args.data); - const newValue = decodeValueArgs(table.valueSchema, { - staticData: newStaticData, - encodedLengths: (previousValue?.__encodedLengths as Hex) ?? "0x", - dynamicData: (previousValue?.__dynamicData as Hex) ?? "0x", - }); - debug("upserting record via splice static", { - namespace: table.namespace, - name: table.name, - key, - previousStaticData, - newStaticData, - previousValue, - newValue, - }); - await tx - .insert(sqlTable) - .values({ - __key: uniqueKey, - __staticData: newStaticData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...key, - ...newValue, - }) - .onConflictDoUpdate({ - target: sqlTable.__key, - set: { - __staticData: newStaticData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...newValue, - }, - }) - .execute(); - } else if (log.eventName === "Store_SpliceDynamicData") { - // TODO: verify that this returns what we expect (doesn't error/undefined on no record) - const previousValue = await tx - .select() - .from(sqlTable) - .where(eq(sqlTable.__key, uniqueKey)) - .execute() - .then( - (rows) => rows[0], - (error) => (error instanceof Error ? error : new Error(String(error))) - ); - if (previousValue instanceof Error) { - // https://github.com/latticexyz/mud/issues/1923 - debug( - "Could not query previous value for splice dynamic data, skipping update", - getTableName(sqlTable), - uniqueKey, - previousValue - ); - continue; - } - const previousDynamicData = (previousValue?.__dynamicData as Hex) ?? "0x"; - const newDynamicData = spliceHex(previousDynamicData, log.args.start, log.args.deleteCount, log.args.data); - const newValue = decodeValueArgs(table.valueSchema, { - staticData: (previousValue?.__staticData as Hex) ?? "0x", - // TODO: handle unchanged encoded lengths - encodedLengths: log.args.encodedLengths, - dynamicData: newDynamicData, - }); - debug("upserting record via splice dynamic", { - namespace: table.namespace, - name: table.name, - key, - previousDynamicData, - newDynamicData, - previousValue, - newValue, - }); - await tx - .insert(sqlTable) - .values({ - __key: uniqueKey, - // TODO: handle unchanged encoded lengths - __encodedLengths: log.args.encodedLengths, - __dynamicData: newDynamicData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...key, - ...newValue, - }) - .onConflictDoUpdate({ - target: sqlTable.__key, - set: { - // TODO: handle unchanged encoded lengths - __encodedLengths: log.args.encodedLengths, - __dynamicData: newDynamicData, - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: false, - ...newValue, - }, - }) - .execute(); - } else if (log.eventName === "Store_DeleteRecord") { - // TODO: should we upsert so we at least have a DB record of when a thing was created/deleted within the same block? - debug("deleting record", { - namespace: table.namespace, - name: table.name, - key, - }); - await tx - .update(sqlTable) - .set({ - __lastUpdatedBlockNumber: blockNumber, - __isDeleted: true, - }) - .where(eq(sqlTable.__key, uniqueKey)) - .execute(); - } - } - - await tx - .insert(internalTables.chain) - .values({ - schemaVersion, - chainId, - lastUpdatedBlockNumber: blockNumber, - }) - .onConflictDoUpdate({ - target: [internalTables.chain.schemaVersion, internalTables.chain.chainId], - set: { - lastUpdatedBlockNumber: blockNumber, - }, - }) - .execute(); - }); - } - - return { - storageAdapter: postgresStorageAdapter, - internalTables, - cleanUp: async (): Promise => { - for (const fn of cleanUp) { - await fn(); - } - }, - }; -} diff --git a/packages/store-sync/src/postgres-decoded/schemaVersion.ts b/packages/store-sync/src/postgres-decoded/schemaVersion.ts deleted file mode 100644 index 397579c6b3..0000000000 --- a/packages/store-sync/src/postgres-decoded/schemaVersion.ts +++ /dev/null @@ -1,4 +0,0 @@ -// When this is incremented, it forces all indexers to reindex from scratch the next time they start up. -// Only use this when the schemas change, until we get proper schema migrations. -// TODO: instead of this, detect schema changes and drop/recreate tables as needed -export const schemaVersion = 1; diff --git a/packages/store-sync/src/postgres-decoded/setupTables.test.ts b/packages/store-sync/src/postgres-decoded/setupTables.test.ts deleted file mode 100644 index 93ab5b7dc8..0000000000 --- a/packages/store-sync/src/postgres-decoded/setupTables.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { buildInternalTables } from "./buildInternalTables"; -import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core"; -import { DefaultLogger } from "drizzle-orm"; -import { drizzle } from "drizzle-orm/postgres-js"; -import postgres from "postgres"; -import { setupTables } from "./setupTables"; -import * as transformSchemaNameExports from "./transformSchemaName"; - -vi.spyOn(transformSchemaNameExports, "transformSchemaName").mockImplementation( - (schemaName) => `${process.pid}_${process.env.VITEST_POOL_ID}__${schemaName}` -); - -describe("setupTables", async () => { - let db: PgDatabase; - const internalTables = buildInternalTables(); - - beforeEach(async () => { - db = drizzle(postgres(process.env.DATABASE_URL!), { - logger: new DefaultLogger(), - }); - }); - - describe("before running", () => { - it("should be missing schemas", async () => { - await expect(db.select().from(internalTables.chain)).rejects.toThrow( - /relation "\w+mud_internal.chain" does not exist/ - ); - await expect(db.select().from(internalTables.tables)).rejects.toThrow( - /relation "\w+mud_internal.tables" does not exist/ - ); - }); - }); - - describe("after running", () => { - beforeEach(async () => { - const cleanUp = await setupTables(db, Object.values(internalTables)); - return cleanUp; - }); - - it("should have schemas", async () => { - expect(await db.select().from(internalTables.chain)).toMatchInlineSnapshot("[]"); - expect(await db.select().from(internalTables.tables)).toMatchInlineSnapshot("[]"); - }); - }); -}); diff --git a/packages/store-sync/src/postgres-decoded/setupTables.ts b/packages/store-sync/src/postgres-decoded/setupTables.ts deleted file mode 100644 index d15f92ce54..0000000000 --- a/packages/store-sync/src/postgres-decoded/setupTables.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { AnyPgColumn, PgTableWithColumns, PgDatabase, getTableConfig } from "drizzle-orm/pg-core"; -import { getTableColumns, sql } from "drizzle-orm"; -import { ColumnDataType } from "kysely"; -import { isDefined } from "@latticexyz/common/utils"; -import { debug } from "./debug"; -import { pgDialect } from "./pgDialect"; - -export async function setupTables( - db: PgDatabase, - tables: PgTableWithColumns[] -): Promise<() => Promise> { - // TODO: add table to internal tables here - // TODO: look up table schema and check if it matches expected schema, drop if not - - const schemaNames = [...new Set(tables.map((table) => getTableConfig(table).schema).filter(isDefined))]; - - await db.transaction(async (tx) => { - for (const schemaName of schemaNames) { - debug(`creating namespace ${schemaName}`); - await tx.execute(sql.raw(pgDialect.schema.createSchema(schemaName).ifNotExists().compile().sql)); - } - - for (const table of tables) { - const tableConfig = getTableConfig(table); - const scopedDb = tableConfig.schema ? pgDialect.withSchema(tableConfig.schema) : pgDialect; - - let query = scopedDb.schema.createTable(tableConfig.name).ifNotExists(); - - const columns = Object.values(getTableColumns(table)) as AnyPgColumn[]; - for (const column of columns) { - query = query.addColumn(column.name, column.getSQLType() as ColumnDataType, (col) => { - if (column.notNull) { - col = col.notNull(); - } - if (column.hasDefault && typeof column.default !== "undefined") { - col = col.defaultTo(column.default); - } - return col; - }); - } - - const primaryKeys = columns.filter((column) => column.primary).map((column) => column.name); - if (primaryKeys.length) { - query = query.addPrimaryKeyConstraint(`${tableConfig.name}__pk`, primaryKeys as any); - } - - debug(`creating table ${tableConfig.name} in namespace ${tableConfig.schema}`); - await tx.execute(sql.raw(query.compile().sql)); - } - }); - - return async () => { - for (const schemaName of schemaNames) { - try { - debug(`dropping namespace ${schemaName} and all of its tables`); - await db.execute(sql.raw(pgDialect.schema.dropSchema(schemaName).ifExists().cascade().compile().sql)); - } catch (error) { - debug(`failed to drop namespace ${schemaName}`, error); - } - } - }; -} diff --git a/packages/store-sync/src/postgres-decoded/syncToPostgres.ts b/packages/store-sync/src/postgres-decoded/syncToPostgres.ts index 10ea4cf2af..e3ac38083f 100644 --- a/packages/store-sync/src/postgres-decoded/syncToPostgres.ts +++ b/packages/store-sync/src/postgres-decoded/syncToPostgres.ts @@ -1,7 +1,7 @@ import { StoreConfig } from "@latticexyz/store"; import { PgDatabase } from "drizzle-orm/pg-core"; import { SyncOptions, SyncResult } from "../common"; -import { postgresStorage } from "./postgresStorage"; +import { postgresStorage } from "./createStorageAdapter"; import { createStoreSync } from "../createStoreSync"; type SyncToPostgresOptions = SyncOptions & { diff --git a/packages/store-sync/src/postgres-decoded/transformSchemaName.ts b/packages/store-sync/src/postgres-decoded/transformSchemaName.ts deleted file mode 100644 index 63693eafbb..0000000000 --- a/packages/store-sync/src/postgres-decoded/transformSchemaName.ts +++ /dev/null @@ -1,4 +0,0 @@ -// This is overridden in tests to better parallelize against the same database -export function transformSchemaName(schemaName: string): string { - return schemaName; -} diff --git a/packages/store-sync/src/postgres/createStorageAdapter.test.ts b/packages/store-sync/src/postgres/createStorageAdapter.test.ts index a378d1931f..554c1315ee 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.test.ts @@ -9,7 +9,6 @@ import { groupLogsByBlockNumber } from "@latticexyz/block-logs-stream"; import { storeEventsAbi } from "@latticexyz/store"; import { StoreEventsLog } from "../common"; import worldRpcLogs from "../../../../test-data/world-logs.json"; -import { tables } from "./tables"; import { resourceToHex } from "@latticexyz/common"; const blocks = groupLogsByBlockNumber( @@ -46,7 +45,7 @@ describe("createStorageAdapter", async () => { await storageAdapter.storageAdapter(block); } - expect(await db.select().from(tables.chainTable)).toMatchInlineSnapshot(` + expect(await db.select().from(storageAdapter.tables.chainTable)).toMatchInlineSnapshot(` [ { "chainId": 31337, @@ -58,8 +57,13 @@ describe("createStorageAdapter", async () => { expect( await db .select() - .from(tables.recordsTable) - .where(eq(tables.recordsTable.tableId, resourceToHex({ type: "table", namespace: "", name: "NumberList" }))) + .from(storageAdapter.tables.recordsTable) + .where( + eq( + storageAdapter.tables.recordsTable.tableId, + resourceToHex({ type: "table", namespace: "", name: "NumberList" }) + ) + ) ).toMatchInlineSnapshot(` [ { diff --git a/packages/store-sync/src/postgres/index.ts b/packages/store-sync/src/postgres/index.ts index ebd661611e..05a54b4890 100644 --- a/packages/store-sync/src/postgres/index.ts +++ b/packages/store-sync/src/postgres/index.ts @@ -1,7 +1,6 @@ export * from "./cleanDatabase"; export * from "./columnTypes"; export * from "./createStorageAdapter"; -export * from "./schemaVersion"; export * from "./setupTables"; export * from "./syncToPostgres"; export * from "./tables"; diff --git a/packages/store-sync/src/postgres/schemaVersion.ts b/packages/store-sync/src/postgres/schemaVersion.ts deleted file mode 100644 index 397579c6b3..0000000000 --- a/packages/store-sync/src/postgres/schemaVersion.ts +++ /dev/null @@ -1,4 +0,0 @@ -// When this is incremented, it forces all indexers to reindex from scratch the next time they start up. -// Only use this when the schemas change, until we get proper schema migrations. -// TODO: instead of this, detect schema changes and drop/recreate tables as needed -export const schemaVersion = 1; diff --git a/packages/store-sync/src/postgres/setupTables.ts b/packages/store-sync/src/postgres/setupTables.ts index 6c43c170ec..94a2ae3762 100644 --- a/packages/store-sync/src/postgres/setupTables.ts +++ b/packages/store-sync/src/postgres/setupTables.ts @@ -38,7 +38,10 @@ export async function setupTables( const primaryKeyColumns = columns.filter((column) => column.primary).map((column) => column.name); if (primaryKeyColumns.length) { - query = query.addPrimaryKeyConstraint(`${primaryKeyColumns.join("_")}_pk`, primaryKeyColumns as any); + query = query.addPrimaryKeyConstraint( + `${tableConfig.name}_${primaryKeyColumns.join("_")}_pk`, + primaryKeyColumns as any + ); } for (const pk of tableConfig.primaryKeys) { @@ -51,7 +54,7 @@ export async function setupTables( for (const index of tableConfig.indexes) { const columnNames = index.config.columns.map((col) => col.name); let query = scopedDb.schema - .createIndex(index.config.name ?? `${columnNames.join("_")}_index`) + .createIndex(index.config.name ?? `${tableConfig.name}_${columnNames.join("_")}_index`) .on(tableConfig.name) .columns(columnNames) .ifNotExists(); From 909cbec25741cac97eac9f1aa0763ba4d99915ec Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 11:10:24 +0000 Subject: [PATCH 11/23] missed a spot --- packages/store-sync/src/postgres-decoded/syncToPostgres.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/store-sync/src/postgres-decoded/syncToPostgres.ts b/packages/store-sync/src/postgres-decoded/syncToPostgres.ts index e3ac38083f..b4c98627e7 100644 --- a/packages/store-sync/src/postgres-decoded/syncToPostgres.ts +++ b/packages/store-sync/src/postgres-decoded/syncToPostgres.ts @@ -1,7 +1,7 @@ import { StoreConfig } from "@latticexyz/store"; import { PgDatabase } from "drizzle-orm/pg-core"; import { SyncOptions, SyncResult } from "../common"; -import { postgresStorage } from "./createStorageAdapter"; +import { createStorageAdapter } from "./createStorageAdapter"; import { createStoreSync } from "../createStoreSync"; type SyncToPostgresOptions = SyncOptions & { @@ -31,7 +31,7 @@ export async function syncToPostgres( startSync = true, ...syncOptions }: SyncToPostgresOptions): Promise { - const { storageAdapter } = await postgresStorage({ database, publicClient, config }); + const { storageAdapter } = await createStorageAdapter({ database, publicClient, config }); const storeSync = await createStoreSync({ storageAdapter, config, From 837ed6bf4deb178277623c88cd3e72a0848c3498 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 11:20:44 +0000 Subject: [PATCH 12/23] update snapshots --- .../src/postgres-decoded/buildTable.test.ts | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/store-sync/src/postgres-decoded/buildTable.test.ts b/packages/store-sync/src/postgres-decoded/buildTable.test.ts index bddf3ff451..72cb488950 100644 --- a/packages/store-sync/src/postgres-decoded/buildTable.test.ts +++ b/packages/store-sync/src/postgres-decoded/buildTable.test.ts @@ -27,10 +27,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__key", + "name": "__key_bytes", "notNull": true, "primaryKey": true, - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -41,12 +41,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__key", + "name": "__key_bytes", "notNull": true, "primary": true, "sqlName": "bytea", "table": [Circular], - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "__lastUpdatedBlockNumber": PgCustomColumn { @@ -223,7 +223,7 @@ describe("buildTable", () => { }, Symbol(drizzle:Name): "users", Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "test_4__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", + Symbol(drizzle:Schema): "test_7__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", Symbol(drizzle:Columns): { "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", @@ -239,10 +239,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__key", + "name": "__key_bytes", "notNull": true, "primaryKey": true, - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -253,12 +253,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__key", + "name": "__key_bytes", "notNull": true, "primary": true, "sqlName": "bytea", "table": [Circular], - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "__lastUpdatedBlockNumber": PgCustomColumn { @@ -468,10 +468,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__key", + "name": "__key_bytes", "notNull": true, "primaryKey": true, - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -482,12 +482,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__key", + "name": "__key_bytes", "notNull": true, "primary": true, "sqlName": "bytea", "table": [Circular], - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "__lastUpdatedBlockNumber": PgCustomColumn { @@ -564,7 +564,7 @@ describe("buildTable", () => { }, Symbol(drizzle:Name): "users", Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "test_4__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", + Symbol(drizzle:Schema): "test_7__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", Symbol(drizzle:Columns): { "__keyBytes": PgCustomColumn { "columnType": "PgCustomColumn", @@ -580,10 +580,10 @@ describe("buildTable", () => { "fieldConfig": undefined, "hasDefault": false, "isUnique": false, - "name": "__key", + "name": "__key_bytes", "notNull": true, "primaryKey": true, - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "dataType": "custom", @@ -594,12 +594,12 @@ describe("buildTable", () => { "isUnique": false, "mapFrom": [Function], "mapTo": [Function], - "name": "__key", + "name": "__key_bytes", "notNull": true, "primary": true, "sqlName": "bytea", "table": [Circular], - "uniqueName": "users___key_unique", + "uniqueName": "users___key_bytes_unique", "uniqueType": undefined, }, "__lastUpdatedBlockNumber": PgCustomColumn { From 276774bf01fc4afaa845c6eebc2a44b50b2a08ad Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 11:57:27 +0000 Subject: [PATCH 13/23] simplify buildTable snapshot test --- .../src/postgres-decoded/buildTable.test.ts | 654 +----------------- 1 file changed, 33 insertions(+), 621 deletions(-) diff --git a/packages/store-sync/src/postgres-decoded/buildTable.test.ts b/packages/store-sync/src/postgres-decoded/buildTable.test.ts index 72cb488950..634cf6bbbe 100644 --- a/packages/store-sync/src/postgres-decoded/buildTable.test.ts +++ b/packages/store-sync/src/postgres-decoded/buildTable.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect } from "vitest"; import { buildTable } from "./buildTable"; +import { getTableColumns } from "drizzle-orm"; +import { getTableConfig } from "drizzle-orm/pg-core"; +import { mapObject } from "@latticexyz/common/utils"; describe("buildTable", () => { it("should create table from schema", async () => { @@ -11,434 +14,46 @@ describe("buildTable", () => { valueSchema: { name: "string", addr: "address" }, }); - expect(table).toMatchInlineSnapshot(` - PgTable { - "__keyBytes": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__key_bytes", - "notNull": true, - "primaryKey": true, - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, + expect(getTableConfig(table).schema).toMatch(/^test_\d+__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test$/); + expect(getTableConfig(table).name).toMatchInlineSnapshot('"users"'); + expect( + mapObject(getTableColumns(table), (column) => ({ + name: column.name, + dataType: column.dataType, + sqlName: column.sqlName, + })) + ).toMatchInlineSnapshot(` + { + "__keyBytes": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "__key_bytes", - "notNull": true, - "primary": true, "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, }, - "__lastUpdatedBlockNumber": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__last_updated_block_number", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, + "__lastUpdatedBlockNumber": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "__last_updated_block_number", - "notNull": false, - "primary": false, "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, }, - "addr": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "addr", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_addr_unique", - "uniqueType": undefined, - }, + "addr": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "addr", - "notNull": true, - "primary": false, "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users_addr_unique", - "uniqueType": undefined, }, - "name": PgText { - "columnType": "PgText", - "config": { - "columnType": "PgText", - "dataType": "string", - "default": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "name", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_name_unique", - "uniqueType": undefined, - }, + "name": { "dataType": "string", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, "name": "name", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users_name_unique", - "uniqueType": undefined, + "sqlName": undefined, }, - "x": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "x", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_x_unique", - "uniqueType": undefined, - }, + "x": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "x", - "notNull": true, - "primary": false, "sqlName": "integer", - "table": [Circular], - "uniqueName": "users_x_unique", - "uniqueType": undefined, }, - "y": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "y", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_y_unique", - "uniqueType": undefined, - }, + "y": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "y", - "notNull": true, - "primary": false, "sqlName": "integer", - "table": [Circular], - "uniqueName": "users_y_unique", - "uniqueType": undefined, }, - Symbol(drizzle:Name): "users", - Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "test_7__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", - Symbol(drizzle:Columns): { - "__keyBytes": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__key_bytes", - "notNull": true, - "primaryKey": true, - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__key_bytes", - "notNull": true, - "primary": true, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, - "__lastUpdatedBlockNumber": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__last_updated_block_number", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__last_updated_block_number", - "notNull": false, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, - "addr": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "addr", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_addr_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "addr", - "notNull": true, - "primary": false, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users_addr_unique", - "uniqueType": undefined, - }, - "name": PgText { - "columnType": "PgText", - "config": { - "columnType": "PgText", - "dataType": "string", - "default": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "name", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_name_unique", - "uniqueType": undefined, - }, - "dataType": "string", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "name": "name", - "notNull": true, - "primary": false, - "table": [Circular], - "uniqueName": "users_name_unique", - "uniqueType": undefined, - }, - "x": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "x", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_x_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "x", - "notNull": true, - "primary": false, - "sqlName": "integer", - "table": [Circular], - "uniqueName": "users_x_unique", - "uniqueType": undefined, - }, - "y": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "y", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_y_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "y", - "notNull": true, - "primary": false, - "sqlName": "integer", - "table": [Circular], - "uniqueName": "users_y_unique", - "uniqueType": undefined, - }, - }, - Symbol(drizzle:BaseName): "users", - Symbol(drizzle:IsAlias): false, - Symbol(drizzle:ExtraConfigBuilder): undefined, - Symbol(drizzle:IsDrizzleTable): true, - Symbol(drizzle:PgInlineForeignKeys): [], } `); }); @@ -452,234 +67,31 @@ describe("buildTable", () => { valueSchema: { addrs: "address[]" }, }); - expect(table).toMatchInlineSnapshot(` - PgTable { - "__keyBytes": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__key_bytes", - "notNull": true, - "primaryKey": true, - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, + expect(getTableConfig(table).schema).toMatch(/^test_\d+__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test$/); + expect(getTableConfig(table).name).toMatchInlineSnapshot('"users"'); + expect( + mapObject(getTableColumns(table), (column) => ({ + name: column.name, + dataType: column.dataType, + sqlName: column.sqlName, + })) + ).toMatchInlineSnapshot(` + { + "__keyBytes": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "__key_bytes", - "notNull": true, - "primary": true, "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, }, - "__lastUpdatedBlockNumber": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__last_updated_block_number", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, + "__lastUpdatedBlockNumber": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "__last_updated_block_number", - "notNull": false, - "primary": false, "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, }, - "addrs": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "addrs", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_addrs_unique", - "uniqueType": undefined, - }, + "addrs": { "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], "name": "addrs", - "notNull": true, - "primary": false, "sqlName": "text", - "table": [Circular], - "uniqueName": "users_addrs_unique", - "uniqueType": undefined, - }, - Symbol(drizzle:Name): "users", - Symbol(drizzle:OriginalName): "users", - Symbol(drizzle:Schema): "test_7__0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF__test", - Symbol(drizzle:Columns): { - "__keyBytes": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__key_bytes", - "notNull": true, - "primaryKey": true, - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__key_bytes", - "notNull": true, - "primary": true, - "sqlName": "bytea", - "table": [Circular], - "uniqueName": "users___key_bytes_unique", - "uniqueType": undefined, - }, - "__lastUpdatedBlockNumber": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "__last_updated_block_number", - "notNull": false, - "primaryKey": false, - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "__last_updated_block_number", - "notNull": false, - "primary": false, - "sqlName": "numeric", - "table": [Circular], - "uniqueName": "users___last_updated_block_number_unique", - "uniqueType": undefined, - }, - "addrs": PgCustomColumn { - "columnType": "PgCustomColumn", - "config": { - "columnType": "PgCustomColumn", - "customTypeParams": { - "dataType": [Function], - "fromDriver": [Function], - "toDriver": [Function], - }, - "dataType": "custom", - "default": undefined, - "fieldConfig": undefined, - "hasDefault": false, - "isUnique": false, - "name": "addrs", - "notNull": true, - "primaryKey": false, - "uniqueName": "users_addrs_unique", - "uniqueType": undefined, - }, - "dataType": "custom", - "default": undefined, - "defaultFn": undefined, - "enumValues": undefined, - "hasDefault": false, - "isUnique": false, - "mapFrom": [Function], - "mapTo": [Function], - "name": "addrs", - "notNull": true, - "primary": false, - "sqlName": "text", - "table": [Circular], - "uniqueName": "users_addrs_unique", - "uniqueType": undefined, - }, }, - Symbol(drizzle:BaseName): "users", - Symbol(drizzle:IsAlias): false, - Symbol(drizzle:ExtraConfigBuilder): undefined, - Symbol(drizzle:IsDrizzleTable): true, - Symbol(drizzle:PgInlineForeignKeys): [], } `); }); From 2b794022be837c4bcc8183b101b262bbb84d48cc Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 06:15:08 -0800 Subject: [PATCH 14/23] Create poor-waves-occur.md --- .changeset/poor-waves-occur.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changeset/poor-waves-occur.md diff --git a/.changeset/poor-waves-occur.md b/.changeset/poor-waves-occur.md new file mode 100644 index 0000000000..0d1f6934ec --- /dev/null +++ b/.changeset/poor-waves-occur.md @@ -0,0 +1,12 @@ +--- +"@latticexyz/common": minor +--- + +Added a `unique` helper to `@latticexyz/common/utils` to narrow an array to its unique elements. + +```ts +import { unique } from "@latticexyz/common/utils"; + +unique([1, 2, 1, 4, 3, 2]); +// [1, 2, 4, 3] +``` From 4263f9f736ea01f8d8902c5fb709b0eae4a42591 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 06:22:39 -0800 Subject: [PATCH 15/23] Create wild-moose-smile.md --- .changeset/wild-moose-smile.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/wild-moose-smile.md diff --git a/.changeset/wild-moose-smile.md b/.changeset/wild-moose-smile.md new file mode 100644 index 0000000000..ad2498ca80 --- /dev/null +++ b/.changeset/wild-moose-smile.md @@ -0,0 +1,7 @@ +--- +"@latticexyz/store-sync": major +--- + +`syncToPostgres` from `@latticexyz/store-sync/postgres` now uses a single table to store all records in their bytes form (`staticData`, `encodedLengths`, and `dynamicData`), more closely mirroring onchain state and enabling more scalability and stability for automatic indexing of many worlds. + +The previous behavior, where schemaful SQL tables are created and populated for each MUD table, has been moved to a separate `@latticexyz/store-sync/postgres-decoded` export bundle. This approach is considered less stable and is intended to be used for analytics purposes rather than hydrating clients. Some previous metadata columns on these tables have been removed in favor of the bytes records table as the source of truth for onchain state. From 79aeb2668aab286b6e7f85702b384d61b0ef399f Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 06:23:52 -0800 Subject: [PATCH 16/23] Update wild-moose-smile.md --- .changeset/wild-moose-smile.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/wild-moose-smile.md b/.changeset/wild-moose-smile.md index ad2498ca80..a9128187f2 100644 --- a/.changeset/wild-moose-smile.md +++ b/.changeset/wild-moose-smile.md @@ -5,3 +5,5 @@ `syncToPostgres` from `@latticexyz/store-sync/postgres` now uses a single table to store all records in their bytes form (`staticData`, `encodedLengths`, and `dynamicData`), more closely mirroring onchain state and enabling more scalability and stability for automatic indexing of many worlds. The previous behavior, where schemaful SQL tables are created and populated for each MUD table, has been moved to a separate `@latticexyz/store-sync/postgres-decoded` export bundle. This approach is considered less stable and is intended to be used for analytics purposes rather than hydrating clients. Some previous metadata columns on these tables have been removed in favor of the bytes records table as the source of truth for onchain state. + +This overhaul is considered breaking and we recommend starting a fresh database when syncing with either of these strategies. From 0aeb05272f326c35dc1b3c4fc2da437abe4faf6c Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 28 Nov 2023 06:34:23 -0800 Subject: [PATCH 17/23] Create seven-rice-dance.md --- .changeset/seven-rice-dance.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/seven-rice-dance.md diff --git a/.changeset/seven-rice-dance.md b/.changeset/seven-rice-dance.md new file mode 100644 index 0000000000..22442dedd2 --- /dev/null +++ b/.changeset/seven-rice-dance.md @@ -0,0 +1,7 @@ +--- +"@latticexyz/store-indexer": minor +--- + +The `findAll` method is now considered deprecated in favor of a new `getLogs` method. This is only implemented in the Postgres indexer for now, with SQLite coming soon. The new `getLogs` method will be an easier and more robust data source to hydrate the client and other indexers and will allow us to add streaming updates from the indexer in the near future. + +For backwards compatibility, `findAll` is now implemented on top of `getLogs`, with record key/value decoding done in memory at request time. This may not scale for large databases, so use wisely. From 061c1d3d4216b0173060a0d4a8f86601dea02cd1 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 11:01:34 +0000 Subject: [PATCH 18/23] remove DEBUG from script in favor of .env --- .gitignore | 2 ++ packages/store-indexer/package.json | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 3f3141c070..1750b5a496 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ yarn-error.log # We don't want projects created from templates to ignore their lockfiles, but we don't # want to check them in here, so we'll ignore them from the root. templates/*/pnpm-lock.yaml + +.env diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json index e1c08d0ba8..8eabd6181c 100644 --- a/packages/store-indexer/package.json +++ b/packages/store-indexer/package.json @@ -26,11 +26,11 @@ "dev": "tsup --watch", "lint": "eslint .", "start:postgres": "concurrently -n indexer,frontend -c cyan,magenta 'tsx bin/postgres-indexer' 'tsx bin/postgres-frontend'", - "start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres", - "start:postgres:testnet": "DEBUG=mud:* DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres", + "start:postgres:local": "DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres", + "start:postgres:testnet": "DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://rpc.holesky.redstone.xyz pnpm start:postgres", "start:sqlite": "tsx bin/sqlite-indexer", - "start:sqlite:local": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite", - "start:sqlite:testnet": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:sqlite", + "start:sqlite:local": "SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite", + "start:sqlite:testnet": "SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://rpc.holesky.redstone.xyz pnpm start:sqlite", "test": "tsc --noEmit --skipLibCheck", "test:ci": "pnpm run test" }, From d860a56bc60b23494a839c4e6a08d745ef6253f1 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 11:24:24 +0000 Subject: [PATCH 19/23] add comments --- .../store-indexer/bin/postgres-indexer.ts | 3 +++ .../store-indexer/src/postgres/getLogs.ts | 21 +++++++++++++++---- .../postgres-decoded/createStorageAdapter.ts | 2 ++ .../src/postgres/createStorageAdapter.ts | 4 ++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index 8812fe5f52..eac1c1a09b 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -48,7 +48,10 @@ try { .select() .from(tables.chainTable) .where(eq(tables.chainTable.chainId, chainId)) + .limit(1) .execute() + // Get the first record in a way that returns a possible `undefined` + // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true` .then((rows) => rows.find(() => true)); if (chainState?.lastUpdatedBlockNumber != null) { diff --git a/packages/store-indexer/src/postgres/getLogs.ts b/packages/store-indexer/src/postgres/getLogs.ts index c39b37aee7..0edd55362a 100644 --- a/packages/store-indexer/src/postgres/getLogs.ts +++ b/packages/store-indexer/src/postgres/getLogs.ts @@ -31,24 +31,37 @@ export async function getLogs( ? [eq(tables.recordsTable.address, address)] : []; + // Query for the block number that the indexer (i.e. chain) is at, in case the + // indexer is further along in the chain than a given store/table's last updated + // block number. We'll then take the highest block number between the indexer's + // chain state and all the records in the query (in case the records updated + // between these queries). Using just the highest block number from the queries + // could potentially signal to the client an older-than-necessary block number, + // for stores/tables that haven't seen recent activity. + // TODO: move the block number query into the records query for atomicity so we don't have to merge them here const chainState = await database .select() .from(tables.chainTable) .where(eq(tables.chainTable.chainId, chainId)) + .limit(1) .execute() + // Get the first record in a way that returns a possible `undefined` + // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true` .then((rows) => rows.find(() => true)); - let blockNumber = chainState?.lastUpdatedBlockNumber ?? 0n; + const indexerBlockNumber = chainState?.lastUpdatedBlockNumber ?? 0n; const records = await database .select() .from(tables.recordsTable) .where(or(...conditions)); - blockNumber = bigIntMax( - blockNumber, - records.reduce((max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n), 0n) + + const blockNumber = records.reduce( + (max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n), + indexerBlockNumber ); const logs = records + // TODO: add this to the query, assuming we can optimize with an index .filter((record) => !record.isDeleted) .map( (record) => diff --git a/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts b/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts index b8e84de9eb..fe5208a4dc 100644 --- a/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres-decoded/createStorageAdapter.ts @@ -84,6 +84,8 @@ export async function createStorageAdapter rows.find(() => true)); if (!record) { const { namespace, name } = hexToResource(log.args.tableId); diff --git a/packages/store-sync/src/postgres/createStorageAdapter.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts index cb9272138b..da587559d4 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -82,6 +82,8 @@ export async function createStorageAdapter rows.find(() => true)); const previousStaticData = previousValue?.staticData ?? "0x"; @@ -129,6 +131,8 @@ export async function createStorageAdapter rows.find(() => true)); const previousDynamicData = previousValue?.dynamicData ?? "0x"; From 5da62473c11d2edb6bf5c17835108e6a213de4d0 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 11:32:30 +0000 Subject: [PATCH 20/23] group logs by table --- packages/common/src/utils/groupBy.ts | 12 ++++++++++++ packages/common/src/utils/index.ts | 1 + .../src/postgres/createQueryAdapter.ts | 14 ++++++++------ 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 packages/common/src/utils/groupBy.ts diff --git a/packages/common/src/utils/groupBy.ts b/packages/common/src/utils/groupBy.ts new file mode 100644 index 0000000000..5a81d49e3f --- /dev/null +++ b/packages/common/src/utils/groupBy.ts @@ -0,0 +1,12 @@ +export function groupBy( + values: readonly value[], + getKey: (value: value) => key +): Map { + const map = new Map(); + for (const value of values) { + const key = getKey(value); + if (!map.has(key)) map.set(key, []); + (map.get(key) as value[]).push(value); + } + return map; +} diff --git a/packages/common/src/utils/index.ts b/packages/common/src/utils/index.ts index 6bf0ddb0d0..f1d4991554 100644 --- a/packages/common/src/utils/index.ts +++ b/packages/common/src/utils/index.ts @@ -4,6 +4,7 @@ export * from "./bigIntMin"; export * from "./bigIntSort"; export * from "./chunk"; export * from "./curry"; +export * from "./groupBy"; export * from "./identity"; export * from "./isDefined"; export * from "./isNotNull"; diff --git a/packages/store-indexer/src/postgres/createQueryAdapter.ts b/packages/store-indexer/src/postgres/createQueryAdapter.ts index 6f2f3b1cbb..c38c21ffbb 100644 --- a/packages/store-indexer/src/postgres/createQueryAdapter.ts +++ b/packages/store-indexer/src/postgres/createQueryAdapter.ts @@ -5,6 +5,7 @@ import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser"; import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer"; import { debug } from "../debug"; import { getLogs } from "./getLogs"; +import { groupBy } from "@latticexyz/common/utils"; /** * Creates a query adapter for the tRPC server/client to query data from Postgres. @@ -27,13 +28,14 @@ export async function createQueryAdapter(database: PgDatabase): Promise `${getAddress(log.address)}:${log.args.tableId}`); + const tablesWithRecords: TableWithRecords[] = tables.map((table) => { - const records = logs - .filter((log) => getAddress(log.address) === getAddress(table.address) && log.args.tableId === table.tableId) - .map((log) => ({ - key: decodeKey(table.keySchema, log.args.keyTuple), - value: decodeValueArgs(table.valueSchema, log.args), - })); + const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? []; + const records = tableLogs.map((log) => ({ + key: decodeKey(table.keySchema, log.args.keyTuple), + value: decodeValueArgs(table.valueSchema, log.args), + })); return { ...table, From 64748cf473713f134ae1116aa43071df3f8367aa Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 11:39:42 +0000 Subject: [PATCH 21/23] add tests --- packages/common/src/utils/groupBy.test.ts | 32 +++++++++++++++++++++++ packages/common/src/utils/unique.test.ts | 15 +++++++++++ 2 files changed, 47 insertions(+) create mode 100644 packages/common/src/utils/groupBy.test.ts create mode 100644 packages/common/src/utils/unique.test.ts diff --git a/packages/common/src/utils/groupBy.test.ts b/packages/common/src/utils/groupBy.test.ts new file mode 100644 index 0000000000..8145aba261 --- /dev/null +++ b/packages/common/src/utils/groupBy.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { groupBy } from "./groupBy"; + +describe("groupBy", () => { + it("should group values by key", () => { + const records = [ + { type: "cat", name: "Bob" }, + { type: "cat", name: "Spot" }, + { type: "dog", name: "Rover" }, + ]; + expect(groupBy(records, (record) => record.type)).toMatchInlineSnapshot(` + Map { + "cat" => [ + { + "name": "Bob", + "type": "cat", + }, + { + "name": "Spot", + "type": "cat", + }, + ], + "dog" => [ + { + "name": "Rover", + "type": "dog", + }, + ], + } + `); + }); +}); diff --git a/packages/common/src/utils/unique.test.ts b/packages/common/src/utils/unique.test.ts new file mode 100644 index 0000000000..67693e0d14 --- /dev/null +++ b/packages/common/src/utils/unique.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { unique } from "./unique"; + +describe("unique", () => { + it("should return unique values", () => { + expect(unique([1, 2, 1, 4, 3, 2])).toMatchInlineSnapshot(` + [ + 1, + 2, + 4, + 3, + ] + `); + }); +}); From 5c29a16288dec1b8a28e877e77a11463313d4d9c Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 03:39:55 -0800 Subject: [PATCH 22/23] Update poor-waves-occur.md --- .changeset/poor-waves-occur.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.changeset/poor-waves-occur.md b/.changeset/poor-waves-occur.md index 0d1f6934ec..2767fc4beb 100644 --- a/.changeset/poor-waves-occur.md +++ b/.changeset/poor-waves-occur.md @@ -2,7 +2,7 @@ "@latticexyz/common": minor --- -Added a `unique` helper to `@latticexyz/common/utils` to narrow an array to its unique elements. +Added `unique` and `groupBy` array helpers to `@latticexyz/common/utils`. ```ts import { unique } from "@latticexyz/common/utils"; @@ -10,3 +10,14 @@ import { unique } from "@latticexyz/common/utils"; unique([1, 2, 1, 4, 3, 2]); // [1, 2, 4, 3] ``` + +```ts +import { groupBy } from "@latticexyz/common/utils"; + +const records = [{ type: "cat", name: "Bob" }, { type: "cat", name: "Spot" }, { type: "dog", name: "Rover" }]; +Object.fromEntries(groupBy(records, (record) => record.type)); +// { +// "cat": [{ type: "cat", name: "Bob" }, { type: "cat", name: "Spot" }], +// "dog: [{ type: "dog", name: "Rover" }] +// } +``` From b02aedc1d229ace5a0c20dfd874e8b092152d8ca Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 29 Nov 2023 11:45:57 +0000 Subject: [PATCH 23/23] prettier --- .changeset/poor-waves-occur.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.changeset/poor-waves-occur.md b/.changeset/poor-waves-occur.md index 2767fc4beb..10b5d563f8 100644 --- a/.changeset/poor-waves-occur.md +++ b/.changeset/poor-waves-occur.md @@ -14,7 +14,11 @@ unique([1, 2, 1, 4, 3, 2]); ```ts import { groupBy } from "@latticexyz/common/utils"; -const records = [{ type: "cat", name: "Bob" }, { type: "cat", name: "Spot" }, { type: "dog", name: "Rover" }]; +const records = [ + { type: "cat", name: "Bob" }, + { type: "cat", name: "Spot" }, + { type: "dog", name: "Rover" }, +]; Object.fromEntries(groupBy(records, (record) => record.type)); // { // "cat": [{ type: "cat", name: "Bob" }, { type: "cat", name: "Spot" }],