From 997286bacafa43bd997c3c752b445acc23726bde Mon Sep 17 00:00:00 2001
From: Andy Cernera <kooshaba@gmail.com>
Date: Fri, 16 Feb 2024 20:55:41 +0900
Subject: [PATCH] feat(store-sync): wait for idle after each chunk of logs in a
 block (#2254)

Co-authored-by: Kevin Ingersoll <kingersoll@gmail.com>
Co-authored-by: yonada <fraserdscott@gmail.com>
---
 .changeset/nasty-rice-pull.md              |  5 +++
 e2e/packages/sync-test/syncToRecs.test.ts  | 47 ++++++++++++++++++++++
 packages/store-sync/src/createStoreSync.ts |  9 ++++-
 3 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 .changeset/nasty-rice-pull.md
 create mode 100644 e2e/packages/sync-test/syncToRecs.test.ts

diff --git a/.changeset/nasty-rice-pull.md b/.changeset/nasty-rice-pull.md
new file mode 100644
index 0000000000..d45fe7daea
--- /dev/null
+++ b/.changeset/nasty-rice-pull.md
@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-sync": minor
+---
+
+`createStoreSync` now [waits for idle](https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback) between each chunk of logs in a block to allow for downstream render cycles to trigger. This means that hydrating logs from an indexer will no longer block until hydration completes, but rather allow for `onProgress` callbacks to trigger.
diff --git a/e2e/packages/sync-test/syncToRecs.test.ts b/e2e/packages/sync-test/syncToRecs.test.ts
new file mode 100644
index 0000000000..7a98bb1ca8
--- /dev/null
+++ b/e2e/packages/sync-test/syncToRecs.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { rpcHttpUrl } from "./setup/constants";
+import { deployContracts } from "./setup";
+import { createAsyncErrorHandler } from "./asyncErrors";
+import { createWorld, getComponentValueStrict } from "@latticexyz/recs";
+import { singletonEntity, syncToRecs } from "@latticexyz/store-sync/recs";
+import mudConfig from "../contracts/mud.config";
+import { transportObserver } from "@latticexyz/common";
+import { ClientConfig, createPublicClient, http, isHex } from "viem";
+import { getNetworkConfig } from "../client-vanilla/src/mud/getNetworkConfig";
+
+describe("syncToRecs", () => {
+  const asyncErrorHandler = createAsyncErrorHandler();
+
+  it("has the correct sync progress percentage", async () => {
+    asyncErrorHandler.resetErrors();
+
+    const { stdout } = await deployContracts(rpcHttpUrl);
+
+    const [, worldAddress] = stdout.match(/worldAddress: '(0x[0-9a-f]+)'/i) ?? [];
+    if (!isHex(worldAddress)) {
+      throw new Error("world address not found in output, did the deploy fail?");
+    }
+
+    const networkConfig = await getNetworkConfig();
+    const clientOptions = {
+      chain: networkConfig.chain,
+      transport: transportObserver(http(rpcHttpUrl ?? undefined)),
+      pollingInterval: 1000,
+    } as const satisfies ClientConfig;
+
+    const publicClient = createPublicClient(clientOptions);
+
+    const world = createWorld();
+
+    const { components } = await syncToRecs({
+      world,
+      config: mudConfig,
+      address: worldAddress,
+      publicClient,
+    });
+
+    expect(getComponentValueStrict(components.SyncProgress, singletonEntity).percentage).toMatchInlineSnapshot(`
+      100
+    `);
+  });
+});
diff --git a/packages/store-sync/src/createStoreSync.ts b/packages/store-sync/src/createStoreSync.ts
index 5bcffb6f89..ad3e2ada04 100644
--- a/packages/store-sync/src/createStoreSync.ts
+++ b/packages/store-sync/src/createStoreSync.ts
@@ -29,7 +29,7 @@ import {
 } from "rxjs";
 import { debug as parentDebug } from "./debug";
 import { SyncStep } from "./SyncStep";
-import { bigIntMax, chunk, isDefined } from "@latticexyz/common/utils";
+import { bigIntMax, chunk, isDefined, waitForIdle } from "@latticexyz/common/utils";
 import { getSnapshot } from "./getSnapshot";
 import { fetchAndStoreLogs } from "./fetchAndStoreLogs";
 
@@ -145,11 +145,16 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
         await storageAdapter({ blockNumber, logs: chunk });
         onProgress?.({
           step: SyncStep.SNAPSHOT,
-          percentage: ((i + chunk.length) / chunks.length) * 100,
+          percentage: ((i + 1) / chunks.length) * 100,
           latestBlockNumber: 0n,
           lastBlockNumberProcessed: blockNumber,
           message: "Hydrating from snapshot",
         });
+
+        // RECS is a synchronous API so hydrating in a loop like this blocks downstream render cycles
+        // that would display the percentage climbing up to 100.
+        // We wait for idle callback here to give rendering a chance to complete.
+        await waitForIdle();
       }
 
       onProgress?.({