feat(store-sync): wait for idle after each chunk of logs in a block (#…

…2254)

Co-authored-by: Kevin Ingersoll <[email protected]>
Co-authored-by: yonada <[email protected]>

.changeset/nasty-rice-pull.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-sync": minor
+---
+
+`createStoreSync` now [waits for idle](https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback) between each chunk of logs in a block to allow for downstream render cycles to trigger. This means that hydrating logs from an indexer will no longer block until hydration completes, but rather allow for `onProgress` callbacks to trigger.

e2e/packages/sync-test/syncToRecs.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { rpcHttpUrl } from "./setup/constants";
+import { deployContracts } from "./setup";
+import { createAsyncErrorHandler } from "./asyncErrors";
+import { createWorld, getComponentValueStrict } from "@latticexyz/recs";
+import { singletonEntity, syncToRecs } from "@latticexyz/store-sync/recs";
+import mudConfig from "../contracts/mud.config";
+import { transportObserver } from "@latticexyz/common";
+import { ClientConfig, createPublicClient, http, isHex } from "viem";
+import { getNetworkConfig } from "../client-vanilla/src/mud/getNetworkConfig";
+
+describe("syncToRecs", () => {
+  const asyncErrorHandler = createAsyncErrorHandler();
+
+  it("has the correct sync progress percentage", async () => {
+    asyncErrorHandler.resetErrors();
+
+    const { stdout } = await deployContracts(rpcHttpUrl);
+
+    const [, worldAddress] = stdout.match(/worldAddress: '(0x[0-9a-f]+)'/i) ?? [];
+    if (!isHex(worldAddress)) {
+      throw new Error("world address not found in output, did the deploy fail?");
+    }
+
+    const networkConfig = await getNetworkConfig();
+    const clientOptions = {
+      chain: networkConfig.chain,
+      transport: transportObserver(http(rpcHttpUrl ?? undefined)),
+      pollingInterval: 1000,
+    } as const satisfies ClientConfig;
+
+    const publicClient = createPublicClient(clientOptions);
+
+    const world = createWorld();
+
+    const { components } = await syncToRecs({
+      world,
+      config: mudConfig,
+      address: worldAddress,
+      publicClient,
+    });
+
+    expect(getComponentValueStrict(components.SyncProgress, singletonEntity).percentage).toMatchInlineSnapshot(`
+      100
+    `);
+  });
+});

packages/store-sync/src/createStoreSync.ts

@@ -29,7 +29,7 @@ import {
 } from "rxjs";
 import { debug as parentDebug } from "./debug";
 import { SyncStep } from "./SyncStep";
-import { bigIntMax, chunk, isDefined } from "@latticexyz/common/utils";
+import { bigIntMax, chunk, isDefined, waitForIdle } from "@latticexyz/common/utils";
 import { getSnapshot } from "./getSnapshot";
 import { fetchAndStoreLogs } from "./fetchAndStoreLogs";
 
@@ -145,11 +145,16 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
         await storageAdapter({ blockNumber, logs: chunk });
         onProgress?.({
           step: SyncStep.SNAPSHOT,
-          percentage: ((i + chunk.length) / chunks.length) * 100,
+          percentage: ((i + 1) / chunks.length) * 100,
           latestBlockNumber: 0n,
           lastBlockNumberProcessed: blockNumber,
           message: "Hydrating from snapshot",
         });
+
+        // RECS is a synchronous API so hydrating in a loop like this blocks downstream render cycles
+        // that would display the percentage climbing up to 100.
+        // We wait for idle callback here to give rendering a chance to complete.
+        await waitForIdle();
       }
 
       onProgress?.({