From 06dff21bda4f623a6ea15cd2e8ef99bd3dd16eeb Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Dec 2024 14:37:35 +0800 Subject: [PATCH 1/3] add text23D plugin, using Hyper3D, which is better than TripoSR --- agent/src/index.ts | 4 + packages/plugin-3d-generation/.npmignore | 7 + .../plugin-3d-generation/eslint.config.mjs | 3 + packages/plugin-3d-generation/package.json | 19 ++ .../plugin-3d-generation/src/constants.ts | 4 + packages/plugin-3d-generation/src/index.ts | 198 ++++++++++++++++++ packages/plugin-3d-generation/tsconfig.json | 15 ++ packages/plugin-3d-generation/tsup.config.ts | 20 ++ 8 files changed, 270 insertions(+) create mode 100644 packages/plugin-3d-generation/.npmignore create mode 100644 packages/plugin-3d-generation/eslint.config.mjs create mode 100644 packages/plugin-3d-generation/package.json create mode 100644 packages/plugin-3d-generation/src/constants.ts create mode 100644 packages/plugin-3d-generation/src/index.ts create mode 100644 packages/plugin-3d-generation/tsconfig.json create mode 100644 packages/plugin-3d-generation/tsup.config.ts diff --git a/agent/src/index.ts b/agent/src/index.ts index 1e49bae84ff..70d430ae3a2 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -46,6 +46,7 @@ import { evmPlugin } from "@elizaos/plugin-evm"; import { storyPlugin } from "@elizaos/plugin-story"; import { flowPlugin } from "@elizaos/plugin-flow"; import { imageGenerationPlugin } from "@elizaos/plugin-image-generation"; +import { ThreeDGenerationPlugin } from "@elizaos/plugin-3d-generation"; import { multiversxPlugin } from "@elizaos/plugin-multiversx"; import { nearPlugin } from "@elizaos/plugin-near"; import { nftGenerationPlugin } from "@elizaos/plugin-nft-generation"; @@ -543,6 +544,9 @@ export async function createAgent( getSecret(character, "HEURIST_API_KEY") ? imageGenerationPlugin : null, + getSecret(character, "FAL_API_KEY") + ? ThreeDGenerationPlugin + : null, ...(getSecret(character, "COINBASE_API_KEY") && getSecret(character, "COINBASE_PRIVATE_KEY") ? [ diff --git a/packages/plugin-3d-generation/.npmignore b/packages/plugin-3d-generation/.npmignore new file mode 100644 index 00000000000..a9227d220f6 --- /dev/null +++ b/packages/plugin-3d-generation/.npmignore @@ -0,0 +1,7 @@ +* + +!dist/** +!package.json +!readme.md +!tsup.config.ts +!tsconfig.json \ No newline at end of file diff --git a/packages/plugin-3d-generation/eslint.config.mjs b/packages/plugin-3d-generation/eslint.config.mjs new file mode 100644 index 00000000000..92fe5bbebef --- /dev/null +++ b/packages/plugin-3d-generation/eslint.config.mjs @@ -0,0 +1,3 @@ +import eslintGlobalConfig from "../../eslint.config.mjs"; + +export default [...eslintGlobalConfig]; diff --git a/packages/plugin-3d-generation/package.json b/packages/plugin-3d-generation/package.json new file mode 100644 index 00000000000..0c14174f022 --- /dev/null +++ b/packages/plugin-3d-generation/package.json @@ -0,0 +1,19 @@ +{ + "name": "@elizaos/plugin-3d-generation", + "version": "0.1.7-alpha.1", + "main": "dist/index.js", + "type": "module", + "types": "dist/index.d.ts", + "dependencies": { + "@elizaos/core": "workspace:*", + "tsup": "8.3.5" + }, + "scripts": { + "build": "tsup --format esm --dts", + "dev": "tsup --format esm --dts --watch", + "lint": "eslint --fix --cache ." + }, + "peerDependencies": { + "whatwg-url": "7.1.0" + } +} diff --git a/packages/plugin-3d-generation/src/constants.ts b/packages/plugin-3d-generation/src/constants.ts new file mode 100644 index 00000000000..dbde2ab6efb --- /dev/null +++ b/packages/plugin-3d-generation/src/constants.ts @@ -0,0 +1,4 @@ +export const FAL_CONSTANTS = { + API_3D_ENDPOINT: "fal-ai/hyper3d/rodin", + API_KEY_SETTING: "FAL_API_KEY", // The setting name to fetch from runtime +}; diff --git a/packages/plugin-3d-generation/src/index.ts b/packages/plugin-3d-generation/src/index.ts new file mode 100644 index 00000000000..67b41c47eaf --- /dev/null +++ b/packages/plugin-3d-generation/src/index.ts @@ -0,0 +1,198 @@ +import { elizaLogger } from "@elizaos/core"; +import { + Action, + HandlerCallback, + IAgentRuntime, + Memory, + Plugin, + State, +} from "@elizaos/core"; +import { fal } from "@fal-ai/client"; +import { FAL_CONSTANTS } from "./constants"; + +import * as fs from 'fs'; +import { Buffer } from 'buffer'; +import * as path from 'path'; +import * as process from 'process'; + +const generate3D = async (prompt: string, runtime: IAgentRuntime) => { + process.env['FAL_KEY'] = FAL_CONSTANTS.API_KEY_SETTING || runtime.getSetting("FAL_API_KEY"); + + try { + elizaLogger.log("Starting 3D generation with prompt:", prompt); + + const response = await fal.subscribe(FAL_CONSTANTS.API_3D_ENDPOINT, { + input: { + prompt: prompt, + input_image_urls: [], + condition_mode: "concat", // fuse concat + geometry_file_format: "glb", // glb usdz fbx obj stl + material: "PBR", // PBR Shaded + quality: "medium", // extra-low, low, medium, high + tier: "Regular" // Regular, Sketch + }, + logs: true, + onQueueUpdate: (update) => { + if (update.status === "IN_PROGRESS") { + update.logs.map((log) => log.message).forEach(elizaLogger.log); + } + }, + }); + + + elizaLogger.log( + "Generation request successful, received response:", + response + ); + + + return {success: true, + url: response.data.model_mesh.url, + file_name: response.data.model_mesh.file_name}; + + } catch (error) { + elizaLogger.error("3D generation error:", error); + return { + success: false, + error: error.message || "Unknown error occurred", + }; + } +}; + +const ThreeDGeneration: Action = { + name: "GENERATE_3D", + similes: [ + "3D_GENERATION", + "3D_GEN", + "CREATE_3D", + "MAKE_3D", + "TEXT23D", + "TEXT_TO_3D", + "3D_CREATE", + "3D_MAKE", + ], + description: "Generate a 3D object based on a text prompt", + validate: async (runtime: IAgentRuntime, _message: Memory) => { + elizaLogger.log("Validating 3D generation action"); + const FalApiKey = runtime.getSetting("FAL_API_KEY"); + elizaLogger.log("FAL_API_KEY present:", !!FalApiKey); + return !!FalApiKey; + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + _state: State, + _options: any, + callback: HandlerCallback + ) => { + elizaLogger.log("3D generation request:", message); + + // Clean up the prompt by removing mentions and commands + const ThreeDPrompt = message.content.text + .replace(/<@\d+>/g, "") // Remove mentions + .replace( + /generate 3D|create 3D|make 3D|render 3D/gi, + "" + ) // Remove commands + .trim(); + + if (!ThreeDPrompt || ThreeDPrompt.length < 3) { + callback({ + text: "Could you please provide more details about what kind of 3D object you'd like me to generate? For example: 'Generate a lovely cat'", + }); + return; + } + + elizaLogger.log("3D prompt:", ThreeDPrompt); + + callback({ + text: `I'll generate a 3D object based on your prompt: "${ThreeDPrompt}". This might take a few minutes...`, + }); + + try { + const result = await generate3D(ThreeDPrompt, runtime); + + if (result.success && result.url && result.file_name) { + // Download the 3D file + const response = await fetch(result.url); + const arrayBuffer = await response.arrayBuffer(); + const ThreeDFileName = `content_cache/generated_3d_${result.file_name}`; + + // ensure the directory is existed + const directoryPath = path.dirname(ThreeDFileName); + if (!fs.existsSync(directoryPath)) { + fs.mkdirSync(directoryPath, { recursive: true }); + } + + // Save 3D file + fs.writeFileSync(ThreeDFileName, Buffer.from(arrayBuffer)); + + callback( + { + text: "Here's your generated 3D object!", + attachments: [ + { + id: crypto.randomUUID(), + url: result.url, + title: "Generated 3D", + source: "ThreeDGeneration", + description: ThreeDPrompt, + text: ThreeDPrompt, + }, + ], + }, + [ThreeDFileName] + ); // Add the 3D file to the attachments + } else { + callback({ + text: `3D generation failed: ${result.error}`, + error: true, + }); + } + } catch (error) { + elizaLogger.error(`Failed to generate 3D. Error: ${error}`); + callback({ + text: `3D generation failed: ${error.message}`, + error: true, + }); + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { text: "Generate a 3D object of a cat playing piano" }, + }, + { + user: "{{agentName}}", + content: { + text: "I'll create a 3D object of a cat playing piano for you", + action: "GENERATE_3D", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can you make a 3D object of a anime character Goku?", + }, + }, + { + user: "{{agentName}}", + content: { + text: "I'll generate a 3D object of a anime character Goku for you", + action: "GENERATE_3D", + }, + }, + ], + ], +} as Action; + +export const ThreeDGenerationPlugin: Plugin = { + name: "3DGeneration", + description: "Generate 3D using Hyper 3D", + actions: [ThreeDGeneration], + evaluators: [], + providers: [], +}; diff --git a/packages/plugin-3d-generation/tsconfig.json b/packages/plugin-3d-generation/tsconfig.json new file mode 100644 index 00000000000..06a0ab4e68f --- /dev/null +++ b/packages/plugin-3d-generation/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../core/tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ] +} \ No newline at end of file diff --git a/packages/plugin-3d-generation/tsup.config.ts b/packages/plugin-3d-generation/tsup.config.ts new file mode 100644 index 00000000000..d6d9bd1a29d --- /dev/null +++ b/packages/plugin-3d-generation/tsup.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + outDir: "dist", + sourcemap: true, + clean: true, + format: ["esm"], + external: [ + "dotenv", + "fs", + "path", + "process", + "@reflink/reflink", + "@node-llama-cpp", + "https", + "http", + "agentkeepalive", + ], +}); From 2e2e074949f1b6250cb9448bbffc3fc014dc9b9e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Dec 2024 14:41:24 +0800 Subject: [PATCH 2/3] add fal ai --- packages/plugin-3d-generation/tsup.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugin-3d-generation/tsup.config.ts b/packages/plugin-3d-generation/tsup.config.ts index d6d9bd1a29d..8c78dce12bf 100644 --- a/packages/plugin-3d-generation/tsup.config.ts +++ b/packages/plugin-3d-generation/tsup.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "process", "@reflink/reflink", "@node-llama-cpp", + "@fal-ai/client", "https", "http", "agentkeepalive", From f477dcf99bd467910b3744a54eeac31ace1587c6 Mon Sep 17 00:00:00 2001 From: odilitime Date: Thu, 26 Dec 2024 01:39:55 +0000 Subject: [PATCH 3/3] add @elizaos/plugin-3d-generation --- agent/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/package.json b/agent/package.json index be8a3e0e290..bb717070184 100644 --- a/agent/package.json +++ b/agent/package.json @@ -52,6 +52,7 @@ "@elizaos/plugin-multiversx": "workspace:*", "@elizaos/plugin-near": "workspace:*", "@elizaos/plugin-zksync-era": "workspace:*", + "@elizaos/plugin-3d-generation": "workspace:*", "readline": "1.3.0", "ws": "8.18.0", "yargs": "17.7.2"