diff --git a/.changeset/thirty-eels-grab.md b/.changeset/thirty-eels-grab.md new file mode 100644 index 0000000000..294b688ec0 --- /dev/null +++ b/.changeset/thirty-eels-grab.md @@ -0,0 +1,9 @@ +--- +"@latticexyz/cli": patch +--- + +Added a `mud pull` command that downloads state from an existing world and uses it to generate a MUD config with tables and system interfaces. This makes it much easier to extend worlds. + +``` +mud pull --worldAddress 0x… --rpc https://… +``` diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index e70e0a89c3..84ffef5bb6 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -14,6 +14,7 @@ import test from "./test"; import trace from "./trace"; import devContracts from "./dev-contracts"; import verify from "./verify"; +import pull from "./pull"; // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Each command has different options export const commands: CommandModule[] = [ @@ -30,4 +31,5 @@ export const commands: CommandModule[] = [ devContracts, abiTs, verify, + pull, ]; diff --git a/packages/cli/src/commands/pull.ts b/packages/cli/src/commands/pull.ts new file mode 100644 index 0000000000..c6b6da8703 --- /dev/null +++ b/packages/cli/src/commands/pull.ts @@ -0,0 +1,74 @@ +import type { CommandModule, InferredOptionTypes } from "yargs"; +import { getRpcUrl } from "@latticexyz/common/foundry"; +import { Address, createClient, http } from "viem"; +import chalk from "chalk"; +import { WriteFileExistsError, pull } from "../pull/pull"; +import path from "node:path"; +import { build } from "../build"; + +const options = { + worldAddress: { type: "string", required: true, desc: "Remote world address" }, + profile: { type: "string", desc: "The foundry profile to use" }, + rpc: { type: "string", desc: "The RPC URL to use. Defaults to the RPC url from the local foundry.toml" }, + rpcBatch: { + type: "boolean", + desc: "Enable batch processing of RPC requests in viem client (defaults to batch size of 100 and wait of 1s)", + }, + replace: { + type: "boolean", + desc: "Replace existing files and directories with data from remote world.", + }, +} as const; + +type Options = InferredOptionTypes; + +const commandModule: CommandModule = { + command: "pull", + + describe: "Pull mud.config.ts and interfaces from an existing world.", + + builder(yargs) { + return yargs.options(options); + }, + + async handler(opts) { + const profile = opts.profile ?? process.env.FOUNDRY_PROFILE; + const rpc = opts.rpc ?? (await getRpcUrl(profile)); + const client = createClient({ + transport: http(rpc, { + batch: opts.rpcBatch + ? { + batchSize: 100, + wait: 1000, + } + : undefined, + }), + }); + + console.log(chalk.bgBlue(chalk.whiteBright(`\n Pulling MUD config from world at ${opts.worldAddress} \n`))); + const rootDir = process.cwd(); + + try { + const { config } = await pull({ + rootDir, + client, + worldAddress: opts.worldAddress as Address, + replace: opts.replace, + }); + await build({ rootDir, config, foundryProfile: profile }); + } catch (error) { + if (error instanceof WriteFileExistsError) { + console.log(); + console.log(chalk.bgRed(chalk.whiteBright(" Error "))); + console.log(` Attempted to write file at "${path.relative(rootDir, error.filename)}", but it already exists.`); + console.log(); + console.log(" To overwrite files, use `--replace` when running this command."); + console.log(); + return; + } + throw error; + } + }, +}; + +export default commandModule; diff --git a/packages/cli/src/deploy/getResourceAccess.ts b/packages/cli/src/deploy/getResourceAccess.ts index 6a793979bc..61ecd4f688 100644 --- a/packages/cli/src/deploy/getResourceAccess.ts +++ b/packages/cli/src/deploy/getResourceAccess.ts @@ -19,6 +19,7 @@ export async function getResourceAccess({ const blockLogs = await fetchBlockLogs({ fromBlock: worldDeploy.deployBlock, toBlock: worldDeploy.stateBlock, + maxBlockRange: 100_000n, async getLogs({ fromBlock, toBlock }) { return getStoreLogs(client, { address: worldDeploy.address, diff --git a/packages/cli/src/deploy/getResourceIds.ts b/packages/cli/src/deploy/getResourceIds.ts index cd050e5f43..d0714c4a2d 100644 --- a/packages/cli/src/deploy/getResourceIds.ts +++ b/packages/cli/src/deploy/getResourceIds.ts @@ -17,6 +17,7 @@ export async function getResourceIds({ const blockLogs = await fetchBlockLogs({ fromBlock: worldDeploy.deployBlock, toBlock: worldDeploy.stateBlock, + maxBlockRange: 100_000n, async getLogs({ fromBlock, toBlock }) { return getStoreLogs(client, { address: worldDeploy.address, diff --git a/packages/cli/src/deploy/getTables.ts b/packages/cli/src/deploy/getTables.ts index f2624dc112..196b24421e 100644 --- a/packages/cli/src/deploy/getTables.ts +++ b/packages/cli/src/deploy/getTables.ts @@ -30,6 +30,7 @@ export async function getTables({ const blockLogs = await fetchBlockLogs({ fromBlock: worldDeploy.deployBlock, toBlock: worldDeploy.stateBlock, + maxBlockRange: 100_000n, async getLogs({ fromBlock, toBlock }) { return getStoreLogs(client, { address: worldDeploy.address, diff --git a/packages/cli/src/pull/debug.ts b/packages/cli/src/pull/debug.ts new file mode 100644 index 0000000000..9fb89d2ba5 --- /dev/null +++ b/packages/cli/src/pull/debug.ts @@ -0,0 +1,10 @@ +import { debug as parentDebug } from "../debug"; + +export const debug = parentDebug.extend("pull"); +export const error = parentDebug.extend("pull"); + +// Pipe debug output to stdout instead of stderr +debug.log = console.debug.bind(console); + +// Pipe error output to stderr +error.log = console.error.bind(console); diff --git a/packages/cli/src/pull/pull.ts b/packages/cli/src/pull/pull.ts new file mode 100644 index 0000000000..8616c6f5d4 --- /dev/null +++ b/packages/cli/src/pull/pull.ts @@ -0,0 +1,235 @@ +import { Address, Client, hexToString, parseAbiItem, stringToHex } from "viem"; +import { getTables } from "../deploy/getTables"; +import { getWorldDeploy } from "../deploy/getWorldDeploy"; +import { getSchemaTypes } from "@latticexyz/protocol-parser/internal"; +import { hexToResource, resourceToHex } from "@latticexyz/common"; +import metadataConfig from "@latticexyz/world-module-metadata/mud.config"; +import { getRecord } from "../deploy/getRecord"; +import path from "node:path"; +import fs from "node:fs/promises"; +import { getResourceIds } from "../deploy/getResourceIds"; +import { getFunctions } from "@latticexyz/world/internal"; +import { abiToInterface, formatSolidity, formatTypescript } from "@latticexyz/common/codegen"; +import { debug } from "./debug"; +import { defineWorld } from "@latticexyz/world"; +import { findUp } from "find-up"; +import { isDefined } from "@latticexyz/common/utils"; + +const ignoredNamespaces = new Set(["store", "world", "metadata"]); + +function namespaceToHex(namespace: string) { + return resourceToHex({ type: "namespace", namespace, name: "" }); +} + +export class WriteFileExistsError extends Error { + name = "WriteFileExistsError"; + constructor(public filename: string) { + super(`Attempted to write file at "${filename}", but it already exists.`); + } +} + +export type PullOptions = { + rootDir: string; + client: Client; + worldAddress: Address; + /** + * Replace existing files and directories with data from remote world. + * Defaults to `true` if `rootDir` is within a git repo, otherwise `false`. + * */ + replace?: boolean; +}; + +export async function pull({ rootDir, client, worldAddress, replace }: PullOptions) { + const replaceFiles = replace ?? (await findUp(".git", { cwd: rootDir })) != null; + + const worldDeploy = await getWorldDeploy(client, worldAddress); + const resourceIds = await getResourceIds({ client, worldDeploy }); + const resources = resourceIds.map(hexToResource).filter((resource) => !ignoredNamespaces.has(resource.namespace)); + const tables = await getTables({ client, worldDeploy }); + + const labels = Object.fromEntries( + ( + await Promise.all( + resourceIds.map(async (resourceId) => { + const { value: bytesValue } = await getRecord({ + client, + worldDeploy, + table: metadataConfig.tables.metadata__ResourceTag, + key: { resource: resourceId, tag: stringToHex("label", { size: 32 }) }, + }); + const value = hexToString(bytesValue); + return [resourceId, value === "" ? null : value]; + }), + ) + ).filter(([, label]) => label != null), + ); + // ensure we always have a root namespace label + labels[namespaceToHex("")] ??= "root"; + + const worldFunctions = await getFunctions({ + client, + worldAddress: worldDeploy.address, + fromBlock: worldDeploy.deployBlock, + toBlock: worldDeploy.stateBlock, + }); + + const namespaces = resources.filter((resource) => resource.type === "namespace"); + const systems = await Promise.all( + resources + .filter((resource) => resource.type === "system") + .map(async ({ namespace, name, resourceId: systemId }) => { + const namespaceId = namespaceToHex(namespace); + // the system name from the system ID can be potentially truncated, so we'll strip off + // any partial "System" suffix and replace it with a full "System" suffix so that it + // matches our criteria for system names + const systemLabel = labels[systemId] ?? name.replace(/(S(y(s(t(e(m)?)?)?)?)?)?$/, "System"); + + const [metadataAbi, metadataWorldAbi] = await Promise.all([ + getRecord({ + client, + worldDeploy, + table: metadataConfig.tables.metadata__ResourceTag, + key: { resource: systemId, tag: stringToHex("abi", { size: 32 }) }, + }) + .then((record) => hexToString(record.value)) + .then((value) => (value !== "" ? value.split("\n") : [])), + getRecord({ + client, + worldDeploy, + table: metadataConfig.tables.metadata__ResourceTag, + key: { resource: systemId, tag: stringToHex("worldAbi", { size: 32 }) }, + }) + .then((record) => hexToString(record.value)) + .then((value) => (value !== "" ? value.split("\n") : [])), + ]); + + const functions = worldFunctions.filter((func) => func.systemId === systemId); + + // If empty or unset ABI in metadata table, backfill with world functions. + // These don't have parameter names or return values, but better than nothing? + const abi = ( + metadataAbi.length ? metadataAbi : functions.map((func) => `function ${func.systemFunctionSignature}`) + ) + .map((sig) => { + try { + return parseAbiItem(sig); + } catch { + debug(`Skipping invalid system signature: ${sig}`); + } + }) + .filter(isDefined); + + const worldAbi = ( + metadataWorldAbi.length ? metadataWorldAbi : functions.map((func) => `function ${func.signature}`) + ) + .map((sig) => { + try { + return parseAbiItem(sig); + } catch { + debug(`Skipping invalid world signature: ${sig}`); + } + }) + .filter(isDefined); + + return { + namespaceId, + namespaceLabel: labels[namespaceId] ?? namespace, + label: systemLabel, + systemId, + namespace, + name, + abi, + worldAbi, + }; + }), + ); + + debug("generating config"); + const configInput = { + namespaces: Object.fromEntries( + namespaces.map(({ namespace, resourceId: namespaceId }) => { + const namespaceLabel = labels[namespaceId] ?? namespace; + return [ + namespaceLabel, + { + ...(namespaceLabel !== namespace ? { namespace } : null), + tables: Object.fromEntries( + tables + .filter((table) => table.namespace === namespace) + .map((table) => { + const tableLabel = labels[table.tableId] ?? table.name; + return [ + tableLabel, + { + ...(tableLabel !== table.name ? { name: table.name } : null), + ...(table.type !== "table" ? { type: table.type } : null), + schema: getSchemaTypes(table.schema), + key: table.key, + deploy: { disabled: true }, + }, + ]; + }), + ), + }, + ]; + }), + ), + }; + + // use the config before writing it so we make sure its valid + // and because we'll use the default paths to write interfaces + debug("validating config"); + const config = defineWorld(configInput); + + debug("writing config"); + await writeFile( + path.join(rootDir, "mud.config.ts"), + await formatTypescript(` + import { defineWorld } from "@latticexyz/world"; + + export default defineWorld(${JSON.stringify(configInput)}); + `), + { overwrite: replaceFiles }, + ); + + const remoteDir = path.join(config.sourceDirectory, "remote"); + if (replaceFiles) { + await fs.rm(remoteDir, { recursive: true, force: true }); + } + + for (const system of systems.filter((system) => system.abi.length)) { + const interfaceName = `I${system.label}`; + const interfaceFile = path.join(remoteDir, "namespaces", system.namespaceLabel, `${interfaceName}.sol`); + + debug("writing system interface", interfaceName, "to", interfaceFile); + const source = abiToInterface({ name: interfaceName, systemId: system.systemId, abi: system.abi }); + await writeFile(path.join(rootDir, interfaceFile), await formatSolidity(source), { overwrite: replaceFiles }); + } + + const worldAbi = systems.flatMap((system) => system.worldAbi); + if (worldAbi.length) { + const interfaceName = "IWorldSystems"; + const interfaceFile = path.join(remoteDir, `${interfaceName}.sol`); + + debug("writing world systems interface to", interfaceFile); + const source = abiToInterface({ name: interfaceName, abi: worldAbi }); + await writeFile(path.join(rootDir, interfaceFile), await formatSolidity(source), { overwrite: replaceFiles }); + } + + return { config }; +} + +export async function exists(filename: string) { + return fs.access(filename).then( + () => true, + () => false, + ); +} + +export async function writeFile(filename: string, contents: string, opts: { overwrite?: boolean } = {}) { + if (!opts.overwrite && (await exists(filename))) { + throw new WriteFileExistsError(filename); + } + await fs.mkdir(path.dirname(filename), { recursive: true }); + await fs.writeFile(filename, contents); +} diff --git a/packages/common/package.json b/packages/common/package.json index 355ac10208..37d60667d1 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -68,6 +68,7 @@ "dependencies": { "@latticexyz/schema-type": "workspace:*", "@solidity-parser/parser": "^0.16.0", + "abitype": "catalog:", "debug": "^4.3.4", "execa": "^7.0.0", "p-queue": "^7.4.1", diff --git a/packages/common/src/codegen/render-solidity/abiToInterface.ts b/packages/common/src/codegen/render-solidity/abiToInterface.ts new file mode 100644 index 0000000000..ce624d696d --- /dev/null +++ b/packages/common/src/codegen/render-solidity/abiToInterface.ts @@ -0,0 +1,60 @@ +import { AbiParameter, Hex } from "viem"; +import { Abi, AbiError, AbiFunction, formatAbiItem, formatAbiParameter } from "abitype"; +import { renderedSolidityHeader } from "./common"; +import { hexToResource } from "../../hexToResource"; + +function formatParam(param: AbiParameter): string { + // return param.type === "string" || param.type === "bytes" || param.type === "tuple" || param.type.endsWith("]") + // ? `${formatAbiParameter(param)} memory` + // : formatAbiParameter(param); + return formatAbiParameter(param); +} + +function formatFunction(item: AbiFunction): string { + const params = item.inputs.map(formatParam).join(", "); + const returns = item.outputs.map(formatParam).join(", "); + return `function ${item.name}(${params}) external${returns.length ? ` returns (${returns})` : ""}`; +} + +function formatSystemId(systemId: Hex): string { + const resource = hexToResource(systemId); + return ` + // equivalent to \`WorldResourceIdLib.encode({ namespace: ${JSON.stringify( + resource.namespace, + )}, name: ${JSON.stringify(resource.name)}, typeId: RESOURCE_SYSTEM });\` + ResourceId constant systemId = ResourceId.wrap(${systemId}); + `; +} + +export type AbiToInterfaceOptions = { + name: string; + systemId?: Hex; + abi: Abi; +}; + +export function abiToInterface({ name, systemId, abi }: AbiToInterfaceOptions): string { + const imports = systemId ? [`{ ResourceId } from "@latticexyz/store/src/ResourceId.sol"`] : []; + const errors = abi.filter((item): item is AbiError => item.type === "error"); + const functions = abi.filter((item): item is AbiFunction => item.type === "function"); + + return ` + ${renderedSolidityHeader} + + ${imports.map((item) => `import ${item};`).join("\n")} + + ${systemId ? formatSystemId(systemId) : ""} + + interface ${name} { + ${errors.map((item) => `${formatAbiItem(item)};`).join("\n")} + + ${functions + .map((item) => { + if ([...item.inputs, ...item.outputs].some((param) => param.type.startsWith("tuple"))) { + return `// TODO: replace tuple with struct\n// ${formatFunction(item)};`; + } + return `${formatFunction(item)};`; + }) + .join("\n")} + } + `; +} diff --git a/packages/common/src/codegen/render-solidity/index.ts b/packages/common/src/codegen/render-solidity/index.ts index 1a9c63f824..931750d7db 100644 --- a/packages/common/src/codegen/render-solidity/index.ts +++ b/packages/common/src/codegen/render-solidity/index.ts @@ -1,3 +1,4 @@ +export * from "./abiToInterface"; export * from "./common"; export * from "./renderEnums"; export * from "./renderImportPath"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f3a7afafa..4717dbc9df 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -311,6 +311,9 @@ importers: '@solidity-parser/parser': specifier: ^0.16.0 version: 0.16.0 + abitype: + specifier: 'catalog:' + version: 1.0.6(typescript@5.4.2)(zod@3.23.8) asn1.js: specifier: 5.x version: 5.4.1 @@ -19853,8 +19856,8 @@ snapshots: is-arguments@1.1.1: dependencies: - call-bind: 1.0.2 - has-tostringtag: 1.0.0 + call-bind: 1.0.7 + has-tostringtag: 1.0.2 is-array-buffer@3.0.4: dependencies: