Skip to content

Commit

Permalink
feat(cli): mud pull (#3171)
Browse files Browse the repository at this point in the history
  • Loading branch information
holic authored Oct 7, 2024
1 parent 03411d8 commit 9e53a51
Show file tree
Hide file tree
Showing 12 changed files with 400 additions and 2 deletions.
9 changes: 9 additions & 0 deletions .changeset/thirty-eels-grab.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
"@latticexyz/cli": patch
---

Added a `mud pull` command that downloads state from an existing world and uses it to generate a MUD config with tables and system interfaces. This makes it much easier to extend worlds.

```
mud pull --worldAddress 0x… --rpc https://…
```
2 changes: 2 additions & 0 deletions packages/cli/src/commands/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import test from "./test";
import trace from "./trace";
import devContracts from "./dev-contracts";
import verify from "./verify";
import pull from "./pull";

// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Each command has different options
export const commands: CommandModule<any, any>[] = [
Expand All @@ -30,4 +31,5 @@ export const commands: CommandModule<any, any>[] = [
devContracts,
abiTs,
verify,
pull,
];
74 changes: 74 additions & 0 deletions packages/cli/src/commands/pull.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import type { CommandModule, InferredOptionTypes } from "yargs";
import { getRpcUrl } from "@latticexyz/common/foundry";
import { Address, createClient, http } from "viem";
import chalk from "chalk";
import { WriteFileExistsError, pull } from "../pull/pull";
import path from "node:path";
import { build } from "../build";

const options = {
worldAddress: { type: "string", required: true, desc: "Remote world address" },
profile: { type: "string", desc: "The foundry profile to use" },
rpc: { type: "string", desc: "The RPC URL to use. Defaults to the RPC url from the local foundry.toml" },
rpcBatch: {
type: "boolean",
desc: "Enable batch processing of RPC requests in viem client (defaults to batch size of 100 and wait of 1s)",
},
replace: {
type: "boolean",
desc: "Replace existing files and directories with data from remote world.",
},
} as const;

type Options = InferredOptionTypes<typeof options>;

const commandModule: CommandModule<Options, Options> = {
command: "pull",

describe: "Pull mud.config.ts and interfaces from an existing world.",

builder(yargs) {
return yargs.options(options);
},

async handler(opts) {
const profile = opts.profile ?? process.env.FOUNDRY_PROFILE;
const rpc = opts.rpc ?? (await getRpcUrl(profile));
const client = createClient({
transport: http(rpc, {
batch: opts.rpcBatch
? {
batchSize: 100,
wait: 1000,
}
: undefined,
}),
});

console.log(chalk.bgBlue(chalk.whiteBright(`\n Pulling MUD config from world at ${opts.worldAddress} \n`)));
const rootDir = process.cwd();

try {
const { config } = await pull({
rootDir,
client,
worldAddress: opts.worldAddress as Address,
replace: opts.replace,
});
await build({ rootDir, config, foundryProfile: profile });
} catch (error) {
if (error instanceof WriteFileExistsError) {
console.log();
console.log(chalk.bgRed(chalk.whiteBright(" Error ")));
console.log(` Attempted to write file at "${path.relative(rootDir, error.filename)}", but it already exists.`);
console.log();
console.log(" To overwrite files, use `--replace` when running this command.");
console.log();
return;
}
throw error;
}
},
};

export default commandModule;
1 change: 1 addition & 0 deletions packages/cli/src/deploy/getResourceAccess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export async function getResourceAccess({
const blockLogs = await fetchBlockLogs({
fromBlock: worldDeploy.deployBlock,
toBlock: worldDeploy.stateBlock,
maxBlockRange: 100_000n,
async getLogs({ fromBlock, toBlock }) {
return getStoreLogs(client, {
address: worldDeploy.address,
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/deploy/getResourceIds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export async function getResourceIds({
const blockLogs = await fetchBlockLogs({
fromBlock: worldDeploy.deployBlock,
toBlock: worldDeploy.stateBlock,
maxBlockRange: 100_000n,
async getLogs({ fromBlock, toBlock }) {
return getStoreLogs(client, {
address: worldDeploy.address,
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/deploy/getTables.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export async function getTables({
const blockLogs = await fetchBlockLogs({
fromBlock: worldDeploy.deployBlock,
toBlock: worldDeploy.stateBlock,
maxBlockRange: 100_000n,
async getLogs({ fromBlock, toBlock }) {
return getStoreLogs(client, {
address: worldDeploy.address,
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/src/pull/debug.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { debug as parentDebug } from "../debug";

export const debug = parentDebug.extend("pull");
export const error = parentDebug.extend("pull");

// Pipe debug output to stdout instead of stderr
debug.log = console.debug.bind(console);

// Pipe error output to stderr
error.log = console.error.bind(console);
235 changes: 235 additions & 0 deletions packages/cli/src/pull/pull.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
import { Address, Client, hexToString, parseAbiItem, stringToHex } from "viem";
import { getTables } from "../deploy/getTables";
import { getWorldDeploy } from "../deploy/getWorldDeploy";
import { getSchemaTypes } from "@latticexyz/protocol-parser/internal";
import { hexToResource, resourceToHex } from "@latticexyz/common";
import metadataConfig from "@latticexyz/world-module-metadata/mud.config";
import { getRecord } from "../deploy/getRecord";
import path from "node:path";
import fs from "node:fs/promises";
import { getResourceIds } from "../deploy/getResourceIds";
import { getFunctions } from "@latticexyz/world/internal";
import { abiToInterface, formatSolidity, formatTypescript } from "@latticexyz/common/codegen";
import { debug } from "./debug";
import { defineWorld } from "@latticexyz/world";
import { findUp } from "find-up";
import { isDefined } from "@latticexyz/common/utils";

const ignoredNamespaces = new Set(["store", "world", "metadata"]);

function namespaceToHex(namespace: string) {
return resourceToHex({ type: "namespace", namespace, name: "" });
}

export class WriteFileExistsError extends Error {
name = "WriteFileExistsError";
constructor(public filename: string) {
super(`Attempted to write file at "${filename}", but it already exists.`);
}
}

export type PullOptions = {
rootDir: string;
client: Client;
worldAddress: Address;
/**
* Replace existing files and directories with data from remote world.
* Defaults to `true` if `rootDir` is within a git repo, otherwise `false`.
* */
replace?: boolean;
};

export async function pull({ rootDir, client, worldAddress, replace }: PullOptions) {
const replaceFiles = replace ?? (await findUp(".git", { cwd: rootDir })) != null;

const worldDeploy = await getWorldDeploy(client, worldAddress);
const resourceIds = await getResourceIds({ client, worldDeploy });
const resources = resourceIds.map(hexToResource).filter((resource) => !ignoredNamespaces.has(resource.namespace));
const tables = await getTables({ client, worldDeploy });

const labels = Object.fromEntries(
(
await Promise.all(
resourceIds.map(async (resourceId) => {
const { value: bytesValue } = await getRecord({
client,
worldDeploy,
table: metadataConfig.tables.metadata__ResourceTag,
key: { resource: resourceId, tag: stringToHex("label", { size: 32 }) },
});
const value = hexToString(bytesValue);
return [resourceId, value === "" ? null : value];
}),
)
).filter(([, label]) => label != null),
);
// ensure we always have a root namespace label
labels[namespaceToHex("")] ??= "root";

const worldFunctions = await getFunctions({
client,
worldAddress: worldDeploy.address,
fromBlock: worldDeploy.deployBlock,
toBlock: worldDeploy.stateBlock,
});

const namespaces = resources.filter((resource) => resource.type === "namespace");
const systems = await Promise.all(
resources
.filter((resource) => resource.type === "system")
.map(async ({ namespace, name, resourceId: systemId }) => {
const namespaceId = namespaceToHex(namespace);
// the system name from the system ID can be potentially truncated, so we'll strip off
// any partial "System" suffix and replace it with a full "System" suffix so that it
// matches our criteria for system names
const systemLabel = labels[systemId] ?? name.replace(/(S(y(s(t(e(m)?)?)?)?)?)?$/, "System");

const [metadataAbi, metadataWorldAbi] = await Promise.all([
getRecord({
client,
worldDeploy,
table: metadataConfig.tables.metadata__ResourceTag,
key: { resource: systemId, tag: stringToHex("abi", { size: 32 }) },
})
.then((record) => hexToString(record.value))
.then((value) => (value !== "" ? value.split("\n") : [])),
getRecord({
client,
worldDeploy,
table: metadataConfig.tables.metadata__ResourceTag,
key: { resource: systemId, tag: stringToHex("worldAbi", { size: 32 }) },
})
.then((record) => hexToString(record.value))
.then((value) => (value !== "" ? value.split("\n") : [])),
]);

const functions = worldFunctions.filter((func) => func.systemId === systemId);

// If empty or unset ABI in metadata table, backfill with world functions.
// These don't have parameter names or return values, but better than nothing?
const abi = (
metadataAbi.length ? metadataAbi : functions.map((func) => `function ${func.systemFunctionSignature}`)
)
.map((sig) => {
try {
return parseAbiItem(sig);
} catch {
debug(`Skipping invalid system signature: ${sig}`);
}
})
.filter(isDefined);

const worldAbi = (
metadataWorldAbi.length ? metadataWorldAbi : functions.map((func) => `function ${func.signature}`)
)
.map((sig) => {
try {
return parseAbiItem(sig);
} catch {
debug(`Skipping invalid world signature: ${sig}`);
}
})
.filter(isDefined);

return {
namespaceId,
namespaceLabel: labels[namespaceId] ?? namespace,
label: systemLabel,
systemId,
namespace,
name,
abi,
worldAbi,
};
}),
);

debug("generating config");
const configInput = {
namespaces: Object.fromEntries(
namespaces.map(({ namespace, resourceId: namespaceId }) => {
const namespaceLabel = labels[namespaceId] ?? namespace;
return [
namespaceLabel,
{
...(namespaceLabel !== namespace ? { namespace } : null),
tables: Object.fromEntries(
tables
.filter((table) => table.namespace === namespace)
.map((table) => {
const tableLabel = labels[table.tableId] ?? table.name;
return [
tableLabel,
{
...(tableLabel !== table.name ? { name: table.name } : null),
...(table.type !== "table" ? { type: table.type } : null),
schema: getSchemaTypes(table.schema),
key: table.key,
deploy: { disabled: true },
},
];
}),
),
},
];
}),
),
};

// use the config before writing it so we make sure its valid
// and because we'll use the default paths to write interfaces
debug("validating config");
const config = defineWorld(configInput);

debug("writing config");
await writeFile(
path.join(rootDir, "mud.config.ts"),
await formatTypescript(`
import { defineWorld } from "@latticexyz/world";
export default defineWorld(${JSON.stringify(configInput)});
`),
{ overwrite: replaceFiles },
);

const remoteDir = path.join(config.sourceDirectory, "remote");
if (replaceFiles) {
await fs.rm(remoteDir, { recursive: true, force: true });
}

for (const system of systems.filter((system) => system.abi.length)) {
const interfaceName = `I${system.label}`;
const interfaceFile = path.join(remoteDir, "namespaces", system.namespaceLabel, `${interfaceName}.sol`);

debug("writing system interface", interfaceName, "to", interfaceFile);
const source = abiToInterface({ name: interfaceName, systemId: system.systemId, abi: system.abi });
await writeFile(path.join(rootDir, interfaceFile), await formatSolidity(source), { overwrite: replaceFiles });
}

const worldAbi = systems.flatMap((system) => system.worldAbi);
if (worldAbi.length) {
const interfaceName = "IWorldSystems";
const interfaceFile = path.join(remoteDir, `${interfaceName}.sol`);

debug("writing world systems interface to", interfaceFile);
const source = abiToInterface({ name: interfaceName, abi: worldAbi });
await writeFile(path.join(rootDir, interfaceFile), await formatSolidity(source), { overwrite: replaceFiles });
}

return { config };
}

export async function exists(filename: string) {
return fs.access(filename).then(
() => true,
() => false,
);
}

export async function writeFile(filename: string, contents: string, opts: { overwrite?: boolean } = {}) {
if (!opts.overwrite && (await exists(filename))) {
throw new WriteFileExistsError(filename);
}
await fs.mkdir(path.dirname(filename), { recursive: true });
await fs.writeFile(filename, contents);
}
1 change: 1 addition & 0 deletions packages/common/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
"dependencies": {
"@latticexyz/schema-type": "workspace:*",
"@solidity-parser/parser": "^0.16.0",
"abitype": "catalog:",
"debug": "^4.3.4",
"execa": "^7.0.0",
"p-queue": "^7.4.1",
Expand Down
Loading

0 comments on commit 9e53a51

Please sign in to comment.