From 85b94614b83cd0964a305d488c1efb247445b915 Mon Sep 17 00:00:00 2001 From: alvarius Date: Thu, 7 Dec 2023 22:11:32 +0100 Subject: [PATCH] feat(store-sync, store-indexer): order logs by logIndex (#2037) --- .changeset/fluffy-days-carry.md | 6 +++++ .../store-indexer/bin/postgres-indexer.ts | 4 +-- packages/store-indexer/src/postgres/common.ts | 3 ++- .../src/postgres/deprecated/getLogs.ts | 11 +++----- .../store-indexer/src/postgres/queryLogs.ts | 7 +++--- .../store-indexer/src/postgres/recordToLog.ts | 4 ++- .../createStorageAdapter.test.ts | 7 +++--- .../src/postgres/createStorageAdapter.test.ts | 7 +++--- .../src/postgres/createStorageAdapter.ts | 25 ++++++++++++------- packages/store-sync/src/postgres/tables.ts | 5 ++-- packages/store-sync/src/postgres/version.ts | 2 +- 11 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 .changeset/fluffy-days-carry.md diff --git a/.changeset/fluffy-days-carry.md b/.changeset/fluffy-days-carry.md new file mode 100644 index 0000000000..fa98b7b4fe --- /dev/null +++ b/.changeset/fluffy-days-carry.md @@ -0,0 +1,6 @@ +--- +"@latticexyz/store-indexer": major +"@latticexyz/store-sync": major +--- + +The postgres indexer is now storing the `logIndex` of the last update of a record to be able to return the snapshot logs in the order they were emitted onchain. diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts index ea17b7d7d7..95d659a280 100644 --- a/packages/store-indexer/bin/postgres-indexer.ts +++ b/packages/store-indexer/bin/postgres-indexer.ts @@ -59,8 +59,8 @@ try { // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true` .then((rows) => rows.find(() => true)); - if (chainState?.lastUpdatedBlockNumber != null) { - startBlock = chainState.lastUpdatedBlockNumber + 1n; + if (chainState?.blockNumber != null) { + startBlock = chainState.blockNumber + 1n; console.log("resuming from block number", startBlock); } } catch (error) { diff --git a/packages/store-indexer/src/postgres/common.ts b/packages/store-indexer/src/postgres/common.ts index ff4807cea7..c8acb890d5 100644 --- a/packages/store-indexer/src/postgres/common.ts +++ b/packages/store-indexer/src/postgres/common.ts @@ -7,7 +7,8 @@ export type RecordData = { staticData: Hex | null; encodedLengths: Hex | null; dynamicData: Hex | null; - lastUpdatedBlockNumber: string; + recordBlockNumber: string; + logIndex: number; }; export type RecordMetadata = { diff --git a/packages/store-indexer/src/postgres/deprecated/getLogs.ts b/packages/store-indexer/src/postgres/deprecated/getLogs.ts index 30896d2d21..e6138fef2e 100644 --- a/packages/store-indexer/src/postgres/deprecated/getLogs.ts +++ b/packages/store-indexer/src/postgres/deprecated/getLogs.ts @@ -55,7 +55,7 @@ export async function getLogs( // Get the first record in a way that returns a possible `undefined` // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true` .then((rows) => rows.find(() => true)); - const indexerBlockNumber = chainState?.lastUpdatedBlockNumber ?? 0n; + const indexerBlockNumber = chainState?.blockNumber ?? 0n; benchmark("query chainState"); const records = await database @@ -63,21 +63,18 @@ export async function getLogs( .from(tables.recordsTable) .where(or(...conditions)) .orderBy( - asc(tables.recordsTable.lastUpdatedBlockNumber) + asc(tables.recordsTable.blockNumber) // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979) ); benchmark("query records"); - const blockNumber = records.reduce( - (max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n), - indexerBlockNumber - ); + const blockNumber = records.reduce((max, record) => bigIntMax(max, record.blockNumber ?? 0n), indexerBlockNumber); benchmark("find block number"); const logs = records // TODO: add this to the query, assuming we can optimize with an index .filter((record) => !record.isDeleted) - .map((record) => recordToLog({ ...record, lastUpdatedBlockNumber: record.lastUpdatedBlockNumber.toString() })); + .map(recordToLog); benchmark("map records to logs"); return { blockNumber, logs }; diff --git a/packages/store-indexer/src/postgres/queryLogs.ts b/packages/store-indexer/src/postgres/queryLogs.ts index e4981234e7..eee4ba8096 100644 --- a/packages/store-indexer/src/postgres/queryLogs.ts +++ b/packages/store-indexer/src/postgres/queryLogs.ts @@ -46,7 +46,7 @@ export function queryLogs(sql: Sql, opts: z.infer): PendingQuery): PendingQuery +): StorageAdapterLog & { eventName: "Store_SetRecord" } { return { address: record.address, eventName: "Store_SetRecord", diff --git a/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts index afe020f4a3..5d266962af 100644 --- a/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres-decoded/createStorageAdapter.test.ts @@ -50,9 +50,9 @@ describe("createStorageAdapter", async () => { expect(await db.select().from(storageAdapter.tables.configTable)).toMatchInlineSnapshot(` [ { + "blockNumber": 12n, "chainId": 31337, - "lastUpdatedBlockNumber": 12n, - "version": "0.0.2", + "version": "0.0.4", }, ] `); @@ -71,13 +71,14 @@ describe("createStorageAdapter", async () => { [ { "address": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342", + "blockNumber": 12n, "dynamicData": "0x000001a400000045", "encodedLengths": "0x0000000000000000000000000000000000000000000000000800000000000008", "isDeleted": false, "key0": null, "key1": null, "keyBytes": "0x", - "lastUpdatedBlockNumber": 12n, + "logIndex": 1, "staticData": null, "tableId": "0x746200000000000000000000000000004e756d6265724c697374000000000000", }, diff --git a/packages/store-sync/src/postgres/createStorageAdapter.test.ts b/packages/store-sync/src/postgres/createStorageAdapter.test.ts index 65599048de..a6b7939543 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.test.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.test.ts @@ -48,9 +48,9 @@ describe("createStorageAdapter", async () => { expect(await db.select().from(storageAdapter.tables.configTable)).toMatchInlineSnapshot(` [ { + "blockNumber": 12n, "chainId": 31337, - "lastUpdatedBlockNumber": 12n, - "version": "0.0.2", + "version": "0.0.4", }, ] `); @@ -69,13 +69,14 @@ describe("createStorageAdapter", async () => { [ { "address": "0x6E9474e9c83676B9A71133FF96Db43E7AA0a4342", + "blockNumber": 12n, "dynamicData": "0x000001a400000045", "encodedLengths": "0x0000000000000000000000000000000000000000000000000800000000000008", "isDeleted": false, "key0": null, "key1": null, "keyBytes": "0x", - "lastUpdatedBlockNumber": 12n, + "logIndex": 1, "staticData": null, "tableId": "0x746200000000000000000000000000004e756d6265724c697374000000000000", }, diff --git a/packages/store-sync/src/postgres/createStorageAdapter.ts b/packages/store-sync/src/postgres/createStorageAdapter.ts index c2a9c9a675..43578e9ad9 100644 --- a/packages/store-sync/src/postgres/createStorageAdapter.ts +++ b/packages/store-sync/src/postgres/createStorageAdapter.ts @@ -54,7 +54,8 @@ export async function createStorageAdapter ({ pk: primaryKey(table.address, table.tableId, table.keyBytes), diff --git a/packages/store-sync/src/postgres/version.ts b/packages/store-sync/src/postgres/version.ts index d2e30349d0..47a7432322 100644 --- a/packages/store-sync/src/postgres/version.ts +++ b/packages/store-sync/src/postgres/version.ts @@ -1 +1 @@ -export const version = "0.0.2"; +export const version = "0.0.4";