Skip to content

Commit

Permalink
Feat: voice chat with AI (#898)
Browse files Browse the repository at this point in the history
* add chats page

* add basic layout and types

* add chat context

* add more components

* add agent form

* add destroy button

* update types

* add models

* update chat agent

* create chat agent

* refactor

* refactor

* add chats CRUD

* notify for chat db update

* refactor

* typo

* chat CRUD

* refactor

* clean code

* add vad

* may record

* may transcribe recording

* update models

* edit chat member

* chat form update

* refactor

* fix chat form

* transcribe in chat

* create chat session

* may create chat session

* update

* update chat

* locale

* refactor

* refactor

* update

* update

* update

* refactor chat

* Fix

* fix

* update prompt

* refactor

* make it works

* update agent message actions

* may assess recording

* fix chat message recording assess

* refine

* refactor

* refactor

* may delete message

* may edit message

* update locales

* fix package issue in Mac

* add destroy callbacks

* fix chats CRUD

* update chats

* add quickstart

* update locales

* refactor

* refactor prompt

* remove console.log

* update

* fix locales
  • Loading branch information
an-lee authored Aug 13, 2024
1 parent 750c0cc commit 1e0297f
Show file tree
Hide file tree
Showing 66 changed files with 5,727 additions and 133 deletions.
7 changes: 6 additions & 1 deletion enjoy/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
"@radix-ui/react-alert-dialog": "^1.1.1",
"@radix-ui/react-aspect-ratio": "^1.1.0",
"@radix-ui/react-avatar": "^1.1.0",
"@radix-ui/react-checkbox": "^1.1.1",
"@radix-ui/react-collapsible": "^1.1.0",
"@radix-ui/react-dropdown-menu": "^2.1.1",
"@radix-ui/react-hover-card": "^1.1.1",
Expand All @@ -112,6 +113,8 @@
"@radix-ui/react-toggle": "^1.1.0",
"@radix-ui/react-tooltip": "^1.1.2",
"@rails/actioncable": "7.1.3",
"@ricky0123/vad-react": "^0.0.24",
"@ricky0123/vad-web": "^0.0.18",
"@sentry/electron": "^5.3.0",
"@uidotdev/usehooks": "^2.4.1",
"@vidstack/react": "^1.11.30",
Expand Down Expand Up @@ -156,7 +159,9 @@
"postcss": "^8.4.40",
"proxy-agent": "^6.4.0",
"react": "^18.3.1",
"react-activity-calendar": "^2.3.1",
"react-activity-calendar": "^2.2.11",
"react-audio-visualize": "^1.1.3",
"react-audio-voice-recorder": "^2.2.0",
"react-dom": "^18.3.1",
"react-hook-form": "^7.52.2",
"react-hotkeys-hook": "^4.5.0",
Expand Down
1 change: 1 addition & 0 deletions enjoy/src/commands/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ export * from "./analyze.command";
export * from "./punctuate.command";
export * from "./summarize-topic.command";
export * from "./text.command";
export * from "./refine.command";
39 changes: 39 additions & 0 deletions enjoy/src/commands/refine.command.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { textCommand } from "./text.command";
import { LANGUAGES } from "@/constants";

export const refineCommand = async (
text: string,
params: {
learningLanguage: string;
nativeLanguage: string;
context: string;
},
options: {
key: string;
modelName?: string;
temperature?: number;
baseUrl?: string;
}
): Promise<string> => {
if (!text) throw new Error("Text is required");

const { learningLanguage, nativeLanguage, context = "None" } = params;
const prompt = await ChatPromptTemplate.fromMessages([
["system", SYSTEM_PROMPT],
["human", text],
]).format({
learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name,
native_language: LANGUAGES.find((l) => l.code === nativeLanguage).name,
context,
});

return textCommand(prompt, options);
};

const SYSTEM_PROMPT = `I speak {native_language}. You're my {learning_language} coach. I'll give you my expression in {learning_language}. And I may also provide some context about my expression.
Please try to understand my true meaning and provide several refined expressions in the native way. And explain them in {native_language}.
[Context]
{context}`;
64 changes: 58 additions & 6 deletions enjoy/src/constants/index.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
export * from './gpt-presets';
export * from './ipa';
export * from "./gpt-presets";
export * from "./ipa";

// https://hf-mirror.com/ggerganov/whisper.cpp/tree/main
import whisperModels from './whisper-models.json';
import whisperModels from "./whisper-models.json";
export const WHISPER_MODELS_OPTIONS = whisperModels;

import languages from './languages.json';
import languages from "./languages.json";
export const LANGUAGES = languages;

export const DATABASE_NAME = "enjoy_database";
Expand All @@ -22,7 +22,8 @@ export const AI_WORKER_ENDPOINT = "https://ai-worker.enjoy.bot";
export const WEB_API_URL = "https://enjoy.bot";
export const WS_URL = "wss://enjoy.bot";

export const REPO_URL = "https://github.com/zuodaotech/everyone-can-use-english";
export const REPO_URL =
"https://github.com/zuodaotech/everyone-can-use-english";

export const SENTRY_DSN =
"https://[email protected]/4506969353289728";
Expand Down Expand Up @@ -55,4 +56,55 @@ export const NOT_SUPPORT_JSON_FORMAT_MODELS = [
"gpt-4-vision-preview",
"gpt-4",
"gpt-4-32k",
];
];

export const CHAT_SYSTEM_PROMPT_TEMPLATE = `You are {name}. {agent_prompt}
You are chatting in an online chat room.
{agent_chat_prompt}
[Rules must be followed]
1. Always reply in {language}.
2. Reply in your personality style and talk in casual way.
3. Reply what you would say only, do not include any other format.
[Chat Topic]
{topic}
[Chat Members]
{members}
[Chat History]
{history}
`;

export const AGENT_FIXTURE_AVA = {
name: "Ava",
introduction: "I'm Ava, your English speaking teacher.",
language: "en-US",
config: {
engine: "enjoyai",
model: "gpt-4o",
prompt:
"You are an experienced English teacher who excels at improving students' speaking skills. You always use simple yet authentic words and sentences to help students understand.",
temperature: 1,
ttsEngine: "enjoyai",
ttsModel: "azure/speech",
ttsVoice: "en-US-AvaNeural",
},
};

export const AGENT_FIXTURE_ANDREW = {
name: "Andrew",
introduction: "I'm Andrew, your American friend.",
language: "en-US",
config: {
engine: "enjoyai",
model: "gpt-4o",
prompt:
"You're a native American who speaks authentic American English, familiar with the culture and customs of the U.S. You're warm and welcoming, eager to make friends from abroad and share all aspects of American life.",
temperature: 0.9,
ttsEngine: "enjoyai",
ttsModel: "azure/speech",
ttsVoice: "en-US-AndrewNeural",
},
};
56 changes: 55 additions & 1 deletion enjoy/src/i18n/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,39 @@
"missingBreak": "There is no break between two words when there is a punctuation between them.",
"monotone": "The word is monotone."
}
},
"chatAgent": {
"notFound": "Chat agent not found",
"created": "Chat agent created",
"updated": "Chat agent updated",
"deleted": "Chat agent deleted",
"name": "Name",
"introduction": "Introduction",
"language": "Language",
"prompt": "Prompt",
"engine": "AI Engine",
"model": "AI Model",
"temperature": "Temperature",
"temperatureDescription": "The higher the temperature, the more creative the result",
"ttsEngine": "TTS Engine",
"ttsModel": "TTS Model",
"ttsVoice": "TTS Voice"
},
"chat": {
"notFound": "Chat not found",
"created": "Chat created",
"updated": "Chat updated",
"deleted": "Chat deleted",
"name": "Name",
"language": "Language",
"topic": "Topic",
"members": "Members",
"memberConfig": "Member Configuration"
}
},
"sidebar": {
"home": "Home",
"chats": "Chats",
"courses": "Courses",
"community": "Community",
"audios": "Audios",
Expand Down Expand Up @@ -660,5 +689,30 @@
"platformInfo": "Platform Information",
"networkState": "Network State",
"connectError": "Connect Error",
"refresh": "refresh"
"refresh": "refresh",
"deleteChat": "Delete chat",
"deleteChatConfirmation": "Are you sure to delete this chat?",
"deleteChatAgent": "Delete agent",
"deleteChatAgentConfirmation": "Are you sure to delete this chat agent?",
"deleteMessage": "Delete message",
"deleteMessageConfirmation": "Are you sure to delete this message?",
"suggestion": "Suggestion",
"editChat": "Edit chat",
"newChat": "New chat",
"addChat": "Add chat",
"agentsManagement": "Agents management",
"newAgent": "New agent",
"editAgent": "Edit agent",
"addToChat": "Add to chat",
"introduction": "Introduction",
"introduceYourself": "Introduce yourself",
"prompt": "Prompt",
"extraPromptForChat": "Extra prompt for this chat",
"promptPreview": "Prompt preview",
"noChatSelected": "No chat selected",
"reRecord": "Re-record",
"quickStart": "Quick start",
"itsYourTurn": "It's your turn. Please start speaking.",
"displayContent": "Display content",
"hideContent": "Hide content"
}
56 changes: 55 additions & 1 deletion enjoy/src/i18n/zh-CN.json
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,39 @@
"missingBreak": "当两个单词之前存在标点符号时,单词之间没有停顿",
"monotone": "这些单词以平淡且不兴奋的语调发音,没有任何节奏或表达"
}
},
"chatAgent": {
"notFound": "未找到角色",
"created": "角色创建成功",
"updated": "角色更新成功",
"deleted": "角色删除成功",
"name": "角色名称",
"introduction": "角色介绍",
"language": "语言",
"prompt": "提示语",
"engine": "AI 引擎",
"model": "AI 模型",
"temperature": "随机性",
"temperatureDescription": "值越高,生成的文本越具创造性,反之则越稳定",
"ttsEngine": "TTS 引擎",
"ttsModel": "TTS 模型",
"ttsVoice": "TTS 音色"
},
"chat": {
"notFound": "未找到聊天",
"created": "聊天创建成功",
"updated": "聊天更新成功",
"deleted": "聊天删除成功",
"name": "聊天标题",
"language": "语言",
"topic": "话题",
"members": "成员",
"memberConfig": "成员配置"
}
},
"sidebar": {
"home": "主页",
"chats": "聊天",
"courses": "课程",
"community": "社区",
"audios": "音频",
Expand Down Expand Up @@ -660,5 +689,30 @@
"platformInfo": "设备信息",
"networkState": "网络状态",
"connectError": "连接错误",
"refresh": "刷新"
"refresh": "刷新",
"deleteChat": "删除聊天",
"deleteChatConfirmation": "您确定要删除此聊天吗?",
"deleteChatAgent": "删除角色",
"deleteChatAgentConfirmation": "您确定要删除此角色吗?",
"deleteMessage": "删除消息",
"deleteMessageConfirmation": "您确定要删除此消息吗?",
"suggestion": "修改建议",
"editChat": "编辑聊天",
"newChat": "新聊天",
"addChat": "添加聊天",
"agentsManagement": "角色管理",
"newAgent": "新角色",
"editAgent": "编辑角色",
"addToChat": "添加到聊天",
"introduction": "介绍",
"introduceYourself": "自我介绍",
"prompt": "提示语",
"extraPromptForChat": "额外提示语",
"promptPreview": "提示语预览",
"noChatSelected": "未选择聊天",
"reRecord": "重新录音",
"quickStart": "快速开始",
"itsYourTurn": "轮到您了,请开始说点什么",
"displayContent": "显示内容",
"hideContent": "隐藏内容"
}
88 changes: 88 additions & 0 deletions enjoy/src/main/db/handlers/chat-agents-handler.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { ipcMain, IpcMainEvent } from "electron";
import { ChatAgent } from "@main/db/models";
import { FindOptions, WhereOptions, Attributes, Op } from "sequelize";
import log from "@main/logger";
import { t } from "i18next";

const logger = log.scope("db/handlers/chat-agents-handler");

class ChatAgentsHandler {
private async findAll(
_event: IpcMainEvent,
options: FindOptions<Attributes<ChatAgent>> & { query?: string }
) {
const { query, where = {} } = options || {};
delete options.query;
delete options.where;

if (query) {
(where as any).name = {
[Op.like]: `%${query}%`,
};
}
const agents = await ChatAgent.findAll({
order: [["name", "ASC"]],
where,
...options,
});

if (!agents) {
return [];
}
return agents.map((agent) => agent.toJSON());
}

private async findOne(
_event: IpcMainEvent,
options: FindOptions<Attributes<ChatAgent>> & {
where: WhereOptions<ChatAgent>;
}
) {
const agent = await ChatAgent.findOne(options);
return agent?.toJSON();
}

private async create(
_event: IpcMainEvent,
data: { name: string; description: string; language: string; config: any }
) {
const agent = await ChatAgent.create(data);
return agent.toJSON();
}

private async update(
_event: IpcMainEvent,
id: string,
data: {
name: string;
description: string;
language: string;
config: any;
}
) {
const agent = await ChatAgent.findByPk(id);
if (!agent) {
throw new Error(t("models.chatAgent.notFound"));
}
await agent.update(data);
return agent.toJSON();
}

private async destroy(_event: IpcMainEvent, id: string) {
const agent = await ChatAgent.findByPk(id);
if (!agent) {
throw new Error(t("models.chatAgent.notFound"));
}
agent.destroy();
}

register() {
ipcMain.handle("chat-agents-find-all", this.findAll);
ipcMain.handle("chat-agents-find-one", this.findOne);
ipcMain.handle("chat-agents-create", this.create);
ipcMain.handle("chat-agents-update", this.update);
ipcMain.handle("chat-agents-destroy", this.destroy);
}
}

export const chatAgentsHandler = new ChatAgentsHandler();
Loading

0 comments on commit 1e0297f

Please sign in to comment.