Merge remote-tracking branch 'origin/main' into holic/indexer-gzip

latticexyz · Aug 15, 2023 · 507f834 · 507f834
2 parents 63725f6 + 5294a7d
commit 507f834
Show file tree

Hide file tree

Showing 36 changed files with 755 additions and 422 deletions.
diff --git a/.changeset/smooth-elephants-wave.md b/.changeset/smooth-elephants-wave.md
@@ -0,0 +1,9 @@
+---
+"@latticexyz/dev-tools": patch
+"@latticexyz/store-indexer": minor
+"@latticexyz/store-sync": minor
+---
+
+Store sync logic is now consolidated into a `createStoreSync` function exported from `@latticexyz/store-sync`. This simplifies each storage sync strategy to just a simple wrapper around the storage adapter. You can now sync to RECS with `syncToRecs` or SQLite with `syncToSqlite` and PostgreSQL support coming soon.
+
+There are no breaking changes if you were just using `syncToRecs` from `@latticexyz/store-sync` or running the `sqlite-indexer` binary from `@latticexyz/store-indexer`.
diff --git a/.changeset/smooth-pots-nail.md b/.changeset/smooth-pots-nail.md
@@ -0,0 +1,6 @@
+---
+"@latticexyz/dev-tools": patch
+"@latticexyz/store-sync": patch
+---
+
+Improves support for internal/client-only RECS components
diff --git a/e2e/packages/client-vanilla/src/mud/setupNetwork.ts b/e2e/packages/client-vanilla/src/mud/setupNetwork.ts
@@ -18,8 +18,6 @@ export async function setupNetwork() {
     pollingInterval: 1000,
   } as const satisfies ClientConfig;
 
-  console.log("client options", clientOptions);
-
   const publicClient = createPublicClient(clientOptions);
 
   const burnerAccount = createBurnerAccount(networkConfig.privateKey as Hex);

diff --git a/e2e/packages/sync-test/setup/startIndexer.ts b/e2e/packages/sync-test/setup/startIndexer.ts
@@ -11,13 +11,17 @@ export function startIndexer(
 ) {
   let resolve: () => void;
   let reject: (reason?: string) => void;
+  const doneSyncing = new Promise<void>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
 
   console.log(chalk.magenta("[indexer]:"), "start syncing");
 
   const proc = execa("pnpm", ["start"], {
     cwd: path.join(__dirname, "..", "..", "..", "..", "packages", "store-indexer"),
     env: {
-      DEBUG: "mud:store-indexer",
+      DEBUG: "mud:*",
       PORT: port.toString(),
       CHAIN_ID: "31337",
       RPC_HTTP_URL: rpcUrl,
@@ -32,31 +36,23 @@ export function startIndexer(
     reject(errorMessage);
   });
 
-  proc.stdout?.on("data", (data) => {
-    const dataString = data.toString();
-    const errors = extractLineContaining("ERROR", dataString).join("\n");
+  function onLog(data: string) {
+    const errors = extractLineContaining("ERROR", data).join("\n");
     if (errors) {
-      console.log(chalk.magenta("[indexer error]:", errors));
-      reject(errors);
-    }
-    console.log(chalk.magentaBright("[indexer]:", dataString));
-  });
-
-  proc.stderr?.on("data", (data) => {
-    const dataString = data.toString();
-    const modeErrors = extractLineContaining("ERROR", dataString).join("\n");
-    if (modeErrors) {
-      const errorMessage = chalk.magenta("[indexer error]:", modeErrors);
+      const errorMessage = chalk.magenta("[indexer error]:", errors);
       console.log(errorMessage);
       reportError(errorMessage);
-      reject(modeErrors);
+      reject(errors);
     }
     if (data.toString().includes("all caught up")) {
       console.log(chalk.magenta("[indexer]:"), "done syncing");
       resolve();
     }
-    console.log(chalk.magentaBright("[indexer ingress]:", dataString));
-  });
+    console.log(chalk.magentaBright("[indexer]:", data));
+  }
+
+  proc.stdout?.on("data", (data) => onLog(data.toString()));
+  proc.stderr?.on("data", (data) => onLog(data.toString()));
 
   function cleanUp() {
     // attempt to clean up sqlite file
@@ -75,10 +71,7 @@ export function startIndexer(
 
   return {
     url: `http://127.0.0.1:${port}/trpc`,
-    doneSyncing: new Promise<void>((res, rej) => {
-      resolve = res;
-      reject = rej;
-    }),
+    doneSyncing,
     process: proc,
     kill: () =>
       new Promise<void>((resolve) => {

diff --git a/examples/minimal/packages/client-phaser/src/ui/LoadingScreen/LoadingScreen.tsx b/examples/minimal/packages/client-phaser/src/ui/LoadingScreen/LoadingScreen.tsx
@@ -4,7 +4,8 @@ import { LoadingBar } from "./LoadingBar";
 import { BootScreen } from "./BootScreen";
 import { useComponentValue } from "@latticexyz/react";
 import { useMUD } from "../../store";
-import { SyncStep, singletonEntity } from "@latticexyz/store-sync/recs";
+import { singletonEntity } from "@latticexyz/store-sync/recs";
+import { SyncStep } from "@latticexyz/store-sync";
 
 export const LoadingScreen = () => {
   const {

diff --git a/packages/dev-tools/src/events/EventIcon.tsx b/packages/dev-tools/src/events/EventIcon.tsx
@@ -1,21 +1,22 @@
 import { assertExhaustive } from "@latticexyz/common/utils";
-import { StoreEventsAbiItem } from "@latticexyz/store";
+import { StoreConfig } from "@latticexyz/store";
+import { StorageOperation } from "@latticexyz/store-sync";
 
 type Props = {
-  eventName: StoreEventsAbiItem["name"];
+  type: StorageOperation<StoreConfig>["type"];
 };
 
-export function EventIcon({ eventName }: Props) {
-  switch (eventName) {
-    case "StoreSetRecord":
+export function EventIcon({ type }: Props) {
+  switch (type) {
+    case "SetRecord":
       return <span className="text-green-500 font-bold">=</span>;
-    case "StoreSetField":
+    case "SetField":
       return <span className="text-cyan-500 font-bold">+</span>;
-    case "StoreDeleteRecord":
+    case "DeleteRecord":
       return <span className="text-red-500 font-bold">-</span>;
-    case "StoreEphemeralRecord":
-      return <span className="text-violet-400 font-bold">~</span>;
+    // case "EphemeralRecord":
+    //   return <span className="text-violet-400 font-bold">~</span>;
     default:
-      return assertExhaustive(eventName, `Unexpected event name: ${eventName}`);
+      return assertExhaustive(type, `Unexpected storage operation type: ${type}`);
   }
 }
diff --git a/packages/dev-tools/src/events/StorageOperationsTable.tsx b/packages/dev-tools/src/events/StorageOperationsTable.tsx
@@ -23,16 +23,23 @@ export function StorageOperationsTable({ operations }: Props) {
       </thead>
       <tbody className="font-mono text-xs">
         {operations.map((operation) => (
-          <tr key={`${operation.log.transactionHash}:${operation.log.transactionIndex}`} className="hover:bg-blue-800">
+          <tr
+            key={
+              operation.log
+                ? `${operation.log.blockHash}:${operation.log.logIndex}`
+                : `${operation.namespace}:${operation.name}:${serialize(operation.key)}`
+            }
+            className="hover:bg-blue-800"
+          >
             <td className="px-1 whitespace-nowrap overflow-hidden text-ellipsis text-white/40">
-              {operation.log.blockNumber.toString()}
+              {operation.log?.blockNumber.toString()}
             </td>
             <td className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
               {operation.namespace}:{operation.name}
             </td>
             <td className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">{serialize(operation.key)}</td>
             <td className="px-1 whitespace-nowrap">
-              <EventIcon eventName={operation.log.eventName} />
+              <EventIcon type={operation.type} />
             </td>
             <td className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
               {operation.type === "SetRecord" ? serialize(operation.value) : null}

diff --git a/packages/dev-tools/src/recs/ComponentData.tsx b/packages/dev-tools/src/recs/ComponentData.tsx
@@ -2,6 +2,7 @@ import { useParams } from "react-router-dom";
 import { useDevToolsContext } from "../DevToolsContext";
 import { ComponentDataTable } from "./ComponentDataTable";
 import { isStoreComponent } from "@latticexyz/store-sync/recs";
+import { StoreComponentDataTable } from "./StoreComponentDataTable";
 
 // TODO: use react-table or similar for better perf with lots of logs
 
@@ -13,9 +14,13 @@ export function ComponentData() {
   const component = world.components.find((component) => component.id === idParam);
 
   // TODO: error message or redirect?
-  if (!component || !isStoreComponent(component)) return null;
+  if (!component) return null;
 
   // key here is useful to force a re-render on component changes,
   // otherwise state hangs around from previous render during navigation (entities)
-  return <ComponentDataTable key={component.id} component={component} />;
+  return isStoreComponent(component) ? (
+    <StoreComponentDataTable key={component.id} component={component} />
+  ) : (
+    <ComponentDataTable key={component.id} component={component} />
+  );
 }
diff --git a/packages/dev-tools/src/recs/ComponentDataTable.tsx b/packages/dev-tools/src/recs/ComponentDataTable.tsx
@@ -1,11 +1,12 @@
 import { useEntityQuery } from "@latticexyz/react";
-import { Component, Has, Schema, getComponentValueStrict } from "@latticexyz/recs";
-import { StoreComponentMetadata, decodeEntity } from "@latticexyz/store-sync/recs";
+import { Component, Has, getComponentValueStrict, Type } from "@latticexyz/recs";
+import { decodeEntity } from "@latticexyz/store-sync/recs";
+import { serialize } from "../serialize";
 
 // TODO: use react-table or similar for better perf with lots of logs
 
 type Props = {
-  component: Component<Schema, StoreComponentMetadata>;
+  component: Component;
 };
 
 export function ComponentDataTable({ component }: Props) {
@@ -16,12 +17,8 @@ export function ComponentDataTable({ component }: Props) {
     <table className="w-full -mx-1">
       <thead className="sticky top-0 z-10 bg-slate-800 text-left">
         <tr className="text-amber-200/80 font-mono">
-          {Object.keys(component.metadata.keySchema).map((name) => (
-            <th key={name} className="px-1 pt-1.5 font-normal">
-              {name}
-            </th>
-          ))}
-          {Object.keys(component.metadata.valueSchema).map((name) => (
+          <th className="px-1 pt-1.5 font-normal">entity</th>
+          {Object.keys(component.schema).map((name) => (
             <th key={name} className="px-1 pt-1.5 font-normal">
               {name}
             </th>
@@ -30,20 +27,19 @@ export function ComponentDataTable({ component }: Props) {
       </thead>
       <tbody className="font-mono text-xs">
         {entities.map((entity) => {
-          const key = decodeEntity(component.metadata.keySchema, entity);
           const value = getComponentValueStrict(component, entity);
           return (
             <tr key={entity}>
-              {Object.keys(component.metadata.keySchema).map((name) => (
-                <td key={name} className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
-                  {String(key[name])}
-                </td>
-              ))}
-              {Object.keys(component.metadata.valueSchema).map((name) => {
+              <td className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">{entity}</td>
+              {Object.keys(component.schema).map((name) => {
                 const fieldValue = value[name];
                 return (
                   <td key={name} className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
-                    {Array.isArray(fieldValue) ? fieldValue.map(String).join(", ") : String(fieldValue)}
+                    {component.schema[name] === Type.T
+                      ? serialize(fieldValue)
+                      : Array.isArray(fieldValue)
+                      ? fieldValue.map(String).join(", ")
+                      : String(fieldValue)}
                   </td>
                 );
               })}

diff --git a/packages/dev-tools/src/recs/ComponentsPage.tsx b/packages/dev-tools/src/recs/ComponentsPage.tsx
@@ -3,25 +3,26 @@ import { NavButton } from "../NavButton";
 import { useEffect, useRef } from "react";
 import { twMerge } from "tailwind-merge";
 import { useDevToolsContext } from "../DevToolsContext";
-import { isStoreComponent } from "@latticexyz/store-sync/recs";
+import { getComponentName } from "./getComponentName";
 
 export function ComponentsPage() {
   const { recsWorld: world } = useDevToolsContext();
   if (!world) throw new Error("Missing recsWorld");
 
-  const components = world.components.filter(isStoreComponent);
+  const components = [...world.components].sort((a, b) => getComponentName(a).localeCompare(getComponentName(b)));
+
   // TODO: lift up selected component so we can remember previous selection between tab nav
   const { id: idParam } = useParams();
-  const selectedComponent = components.find((component) => component.id === idParam);
+  const selectedComponent = components.find((component) => component.id === idParam) ?? components[0];
 
   const detailsRef = useRef<HTMLDetailsElement>(null);
   const navigate = useNavigate();
 
   useEffect(() => {
-    if (components.length && !selectedComponent) {
-      navigate(components[0].id);
+    if (idParam !== selectedComponent.id) {
+      navigate(selectedComponent.id);
     }
-  }, [components, selectedComponent]);
+  }, [idParam, selectedComponent.id]);
 
   useEffect(() => {
     const listener = (event: MouseEvent) => {
@@ -45,7 +46,7 @@ export function ComponentsPage() {
             <summary className="group pointer-events-auto cursor-pointer inline-flex">
               <span className="inline-flex gap-2 px-3 py-2 items-center border-2 border-white/10 rounded group-hover:border-blue-700 group-hover:bg-blue-700 group-hover:text-white">
                 {selectedComponent ? (
-                  <span className="font-mono">{selectedComponent.metadata.componentName}</span>
+                  <span className="font-mono">{getComponentName(selectedComponent)}</span>
                 ) : (
                   <span>Pick a component…</span>
                 )}
@@ -68,7 +69,7 @@ export function ComponentsPage() {
                       }
                     }}
                   >
-                    {component.metadata.componentName}
+                    {getComponentName(component)}
                   </NavButton>
                 ))}
               </div>

diff --git a/packages/dev-tools/src/recs/StoreComponentDataTable.tsx b/packages/dev-tools/src/recs/StoreComponentDataTable.tsx
@@ -0,0 +1,56 @@
+import { useEntityQuery } from "@latticexyz/react";
+import { Component, Has, Schema, getComponentValueStrict } from "@latticexyz/recs";
+import { StoreComponentMetadata, decodeEntity } from "@latticexyz/store-sync/recs";
+
+// TODO: use react-table or similar for better perf with lots of logs
+
+type Props = {
+  component: Component<Schema, StoreComponentMetadata>;
+};
+
+export function StoreComponentDataTable({ component }: Props) {
+  // TODO: this breaks when navigating because its state still has entity IDs from prev page
+  const entities = useEntityQuery([Has(component)]);
+
+  return (
+    <table className="w-full -mx-1">
+      <thead className="sticky top-0 z-10 bg-slate-800 text-left">
+        <tr className="text-amber-200/80 font-mono">
+          {Object.keys(component.metadata.keySchema).map((name) => (
+            <th key={name} className="px-1 pt-1.5 font-normal">
+              {name}
+            </th>
+          ))}
+          {Object.keys(component.metadata.valueSchema).map((name) => (
+            <th key={name} className="px-1 pt-1.5 font-normal">
+              {name}
+            </th>
+          ))}
+        </tr>
+      </thead>
+      <tbody className="font-mono text-xs">
+        {entities.map((entity) => {
+          const key = decodeEntity(component.metadata.keySchema, entity);
+          const value = getComponentValueStrict(component, entity);
+          return (
+            <tr key={entity}>
+              {Object.keys(component.metadata.keySchema).map((name) => (
+                <td key={name} className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
+                  {String(key[name])}
+                </td>
+              ))}
+              {Object.keys(component.metadata.valueSchema).map((name) => {
+                const fieldValue = value[name];
+                return (
+                  <td key={name} className="px-1 whitespace-nowrap overflow-hidden text-ellipsis">
+                    {Array.isArray(fieldValue) ? fieldValue.map(String).join(", ") : String(fieldValue)}
+                  </td>
+                );
+              })}
+            </tr>
+          );
+        })}
+      </tbody>
+    </table>
+  );
+}
diff --git a/packages/dev-tools/src/recs/getComponentName.ts b/packages/dev-tools/src/recs/getComponentName.ts
@@ -0,0 +1,5 @@
+import { Component } from "@latticexyz/recs";
+
+export function getComponentName(component: Component): string {
+  return String(component.metadata?.componentName ?? component.id);
+}
diff --git a/packages/dev-tools/src/summary/ComponentsSummary.tsx b/packages/dev-tools/src/summary/ComponentsSummary.tsx
@@ -1,25 +1,25 @@
 import { World } from "@latticexyz/recs";
 import { NavButton } from "../NavButton";
-import { isStoreComponent } from "@latticexyz/store-sync/recs";
+import { getComponentName } from "../recs/getComponentName";
 
 type Props = {
   world: World;
 };
 
 export function ComponentsSummary({ world }: Props) {
-  const componentsWithName = world.components.filter(isStoreComponent);
+  const components = [...world.components].sort((a, b) => getComponentName(a).localeCompare(getComponentName(b)));
   return (
     <>
-      {componentsWithName.length ? (
+      {components.length ? (
         <>
           <div className="flex flex-col gap-1 items-start">
-            {componentsWithName.map((component) => (
+            {components.map((component) => (
               <NavButton
                 key={component.id}
                 to={`/components/${component.id}`}
                 className="font-mono text-xs hover:text-white"
               >
-                {String(component.metadata.componentName)}
+                {getComponentName(component)}
               </NavButton>
             ))}
           </div>