From 276f4611c860aed5c8c3e85cea8b66fc3afcf390 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:28:54 -0800 Subject: [PATCH 1/7] add Livepeer configuration to .env.example --- .env.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 1c171bffdb0..fb5a98cc27e 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,10 @@ SMALL_HYPERBOLIC_MODEL= # Default: meta-llama/Llama-3.2-3B-Instruct MEDIUM_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-70B-Instruct LARGE_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-405-Instruct +# Livepeer configuration +LIVEPEER_GATEWAY_URL= # Free inference gateways and docs: https://livepeer-eliza.com/ +LIVEPEER_IMAGE_MODEL= # Default: ByteDance/SDXL-Lightning + # Speech Synthesis ELEVENLABS_XI_API_KEY= # API key from elevenlabs From 025feb9c23c2f300d380839fa2ad4fc506f5f302 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:30:11 -0800 Subject: [PATCH 2/7] add Livepeer gateway URL to agent configuration --- agent/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agent/src/index.ts b/agent/src/index.ts index 11bd3e7148a..a76079cdbfd 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -554,7 +554,8 @@ export async function createAgent( getSecret(character, "FAL_API_KEY") || getSecret(character, "OPENAI_API_KEY") || getSecret(character, "VENICE_API_KEY") || - getSecret(character, "HEURIST_API_KEY") + getSecret(character, "HEURIST_API_KEY") || + getSecret(character, "LIVEPEER_GATEWAY_URL") ? imageGenerationPlugin : null, getSecret(character, "FAL_API_KEY") ? ThreeDGenerationPlugin : null, From b27ae3901e271a8fa2012908510f13189fe30a50 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:31:47 -0800 Subject: [PATCH 3/7] add Livepeer model configuration --- packages/core/src/models.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index 53d3b8be244..32901043841 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -486,6 +486,23 @@ export const models: Models = { "Meta-Llama-3-1-405B-Instruct-FP8", }, }, + [ModelProviderName.LIVEPEER]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + // livepeer endpoint is handled from the sdk + model: { + [ModelClass.SMALL]: "", + [ModelClass.MEDIUM]: "", + [ModelClass.LARGE]: "", + [ModelClass.EMBEDDING]: "", + [ModelClass.IMAGE]: settings.LIVEPEER_IMAGE_MODEL || "ByteDance/SDXL-Lightning", + }, + }, }; export function getModel(provider: ModelProviderName, type: ModelClass) { From 698a3eb5a111e5374237a400cfd189f9b7e319f8 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:32:04 -0800 Subject: [PATCH 4/7] add Livepeer model and provider name to core types --- packages/core/src/types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 6786bb99f9e..b16d400240c 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -209,6 +209,7 @@ export type Models = { [ModelProviderName.HYPERBOLIC]: Model; [ModelProviderName.VENICE]: Model; [ModelProviderName.AKASH_CHAT_API]: Model; + [ModelProviderName.LIVEPEER]: Model; }; /** @@ -238,6 +239,7 @@ export enum ModelProviderName { HYPERBOLIC = "hyperbolic", VENICE = "venice", AKASH_CHAT_API = "akash_chat_api", + LIVEPEER = "livepeer", } /** From 57c55416b9d77b6030fa48eb5c2ed9d0c65459d8 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:32:40 -0800 Subject: [PATCH 5/7] Included image generation action to include validation for Livepeer gateway URL --- packages/plugin-image-generation/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts index bf65aacc667..3fbcce20123 100644 --- a/packages/plugin-image-generation/src/index.ts +++ b/packages/plugin-image-generation/src/index.ts @@ -86,6 +86,7 @@ const imageGeneration: Action = { const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY"); const openAiApiKeyOk = !!runtime.getSetting("OPENAI_API_KEY"); const veniceApiKeyOk = !!runtime.getSetting("VENICE_API_KEY"); + const livepeerGatewayUrlOk = !!runtime.getSetting("LIVEPEER_GATEWAY_URL"); return ( anthropicApiKeyOk || @@ -93,7 +94,8 @@ const imageGeneration: Action = { heuristApiKeyOk || falApiKeyOk || openAiApiKeyOk || - veniceApiKeyOk + veniceApiKeyOk || + livepeerGatewayUrlOk ); }, handler: async ( From 74c85a27eb3de37e02fc6708ea0abc15915db5ef Mon Sep 17 00:00:00 2001 From: Titan Node Date: Fri, 27 Dec 2024 23:33:19 -0800 Subject: [PATCH 6/7] Implement Livepeer image generation logic with URL validation --- packages/core/src/generation.ts | 61 ++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index 033720b5a9a..4add3535d1e 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -978,13 +978,16 @@ export const generateImage = async ( return runtime.getSetting("OPENAI_API_KEY"); case ModelProviderName.VENICE: return runtime.getSetting("VENICE_API_KEY"); + case ModelProviderName.LIVEPEER: + return runtime.getSetting("LIVEPEER_GATEWAY_URL"); default: // If no specific match, try the fallback chain return (runtime.getSetting("HEURIST_API_KEY") ?? runtime.getSetting("TOGETHER_API_KEY") ?? runtime.getSetting("FAL_API_KEY") ?? runtime.getSetting("OPENAI_API_KEY") ?? - runtime.getSetting("VENICE_API_KEY")); + runtime.getSetting("VENICE_API_KEY"))?? + runtime.getSetting("LIVEPEER_GATEWAY_URL"); } })(); try { @@ -1176,6 +1179,62 @@ export const generateImage = async ( }); return { success: true, data: base64s }; + + } else if (runtime.imageModelProvider === ModelProviderName.LIVEPEER) { + if (!apiKey) { + throw new Error("Livepeer Gateway is not defined"); + } + try { + const baseUrl = new URL(apiKey); + if (!baseUrl.protocol.startsWith('http')) { + throw new Error("Invalid Livepeer Gateway URL protocol"); + } + const response = await fetch(`${baseUrl.toString()}text-to-image`, { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + model_id: data.modelId || "ByteDance/SDXL-Lightning", + prompt: data.prompt, + width: data.width || 1024, + height: data.height || 1024 + }) + }); + const result = await response.json(); + if (!result.images?.length) { + throw new Error("No images generated"); + } + const base64Images = await Promise.all( + result.images.map(async (image) => { + console.log("imageUrl console log", image.url); + let imageUrl; + if (image.url.includes("http")) { + imageUrl = image.url; + } else { + imageUrl = `${apiKey}${image.url}`; + } + const imageResponse = await fetch(imageUrl); + if (!imageResponse.ok) { + throw new Error( + `Failed to fetch image: ${imageResponse.statusText}` + ); + } + const blob = await imageResponse.blob(); + const arrayBuffer = await blob.arrayBuffer(); + const base64 = Buffer.from(arrayBuffer).toString("base64"); + return `data:image/jpeg;base64,${base64}`; + }) + ); + return { + success: true, + data: base64Images + }; + } catch (error) { + console.error(error); + return { success: false, error: error }; + } + } else { let targetSize = `${data.width}x${data.height}`; if ( From 74f8984b9fe72f80a912eccea127060ee41c9422 Mon Sep 17 00:00:00 2001 From: Titan Node Date: Sat, 28 Dec 2024 00:43:12 -0800 Subject: [PATCH 7/7] Add Livepeer model and configuration details to documentation - Updated ModelProviderName.md to include Livepeer as a model provider. - Added Livepeer model alias in Models.md. - Included Livepeer gateway URL in quickstart.md and configuration.md. - Enhanced documentation to guide users on setting up Livepeer models. --- docs/api/enumerations/ModelProviderName.md | 10 ++++++++++ docs/api/type-aliases/Models.md | 4 ++++ docs/docs/api/enumerations/ModelProviderName.md | 10 ++++++++++ docs/docs/api/type-aliases/Models.md | 4 ++++ docs/docs/guides/configuration.md | 3 +++ docs/docs/quickstart.md | 6 ++++-- 6 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/api/enumerations/ModelProviderName.md b/docs/api/enumerations/ModelProviderName.md index 6707ec27521..01dd0ce401c 100644 --- a/docs/api/enumerations/ModelProviderName.md +++ b/docs/api/enumerations/ModelProviderName.md @@ -233,3 +233,13 @@ Available model providers #### Defined in [packages/core/src/types.ts:240](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L240) + +*** + +### LIVEPEER + +> **LIVEPEER**: `"livepeer"` + +#### Defined in + +[packages/core/src/types.ts:241](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L241) diff --git a/docs/api/type-aliases/Models.md b/docs/api/type-aliases/Models.md index cd9efa078bd..2a5bdc2736a 100644 --- a/docs/api/type-aliases/Models.md +++ b/docs/api/type-aliases/Models.md @@ -100,6 +100,10 @@ Model configurations by provider > **akash\_chat\_api**: [`Model`](Model.md) +### livepeer + +> **livepeer**: [`Model`](Model.md) + ## Defined in [packages/core/src/types.ts:188](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L188) diff --git a/docs/docs/api/enumerations/ModelProviderName.md b/docs/docs/api/enumerations/ModelProviderName.md index 329b4f0c4a1..df08f757721 100644 --- a/docs/docs/api/enumerations/ModelProviderName.md +++ b/docs/docs/api/enumerations/ModelProviderName.md @@ -119,3 +119,13 @@ #### Defined in [packages/core/src/types.ts:132](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L132) + +--- + +### LIVEPEER + +> **LIVEPEER**: `"livepeer"` + +#### Defined in + +[packages/core/src/types.ts:133](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L133) diff --git a/docs/docs/api/type-aliases/Models.md b/docs/docs/api/type-aliases/Models.md index 9ac7fa9b6a3..0331afba238 100644 --- a/docs/docs/api/type-aliases/Models.md +++ b/docs/docs/api/type-aliases/Models.md @@ -52,6 +52,10 @@ > **heurist**: [`Model`](Model.md) +### livepeer + +> **livepeer**: [`Model`](Model.md) + ## Defined in [packages/core/src/types.ts:105](https://github.com/elizaos/eliza/blob/7fcf54e7fb2ba027d110afcc319c0b01b3f181dc/packages/core/src/types.ts#L105) diff --git a/docs/docs/guides/configuration.md b/docs/docs/guides/configuration.md index 20ae3238865..3b9a5e50a69 100644 --- a/docs/docs/guides/configuration.md +++ b/docs/docs/guides/configuration.md @@ -72,6 +72,9 @@ TOGETHER_API_KEY= # Heurist Settings HEURIST_API_KEY= +# Livepeer Settings +LIVEPEER_GATEWAY_URL= + # Local Model Settings XAI_MODEL=meta-llama/Llama-3.1-7b-instruct ``` diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md index 7f9dc9ed4b6..f04d38d92f8 100644 --- a/docs/docs/quickstart.md +++ b/docs/docs/quickstart.md @@ -82,6 +82,7 @@ pnpm build OPENAI_API_KEY= # OpenAI API key GROK_API_KEY= # Grok API key ELEVENLABS_XI_API_KEY= # API key from elevenlabs (for voice) + LIVEPEER_GATEWAY_URL= # Livepeer gateway URL ``` ## Choose Your Model @@ -94,6 +95,7 @@ Eliza supports multiple AI models: - **Llama**: Set `XAI_MODEL=meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` - **Grok**: Set `XAI_MODEL=grok-beta` - **OpenAI**: Set `XAI_MODEL=gpt-4o-mini` or `gpt-4o` +- **Livepeer**: Set `LIVEPEER_IMAGE_MODEL` to your chosen Livepeer image model, available models [here](https://livepeer-eliza.com/) You set which model to use inside the character JSON file @@ -216,8 +218,8 @@ pnpm start --characters="characters/trump.character.json,characters/tate.charact - Ensure Node.js 23.3.0 is installed - Use `node -v` to check version - Consider using [nvm](https://github.com/nvm-sh/nvm) to manage Node versions - - NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use + + NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use ```bash pnpm env use --global 23.3.0 ```