Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat multi models #4457

Merged
merged 12 commits into from
Apr 8, 2024
Prev Previous commit
Next Next commit
feat: roles must alternate between user and assistant in claude, so a…
…dd a fake assistant message between two user messages
butterfly committed Apr 7, 2024
commit 86b5c5585523c042a0a2ab451a5bfa50dd95872c
172 changes: 20 additions & 152 deletions app/client/platforms/anthropic.ts
Original file line number Diff line number Diff line change
@@ -69,31 +69,21 @@ const ClaudeMapper = {
system: "user",
} as const;

const keys = ["claude-2, claude-instant-1"];

export class ClaudeApi implements LLMApi {
extractMessage(res: any) {
console.log("[Response] claude response: ", res);

return res.completion;
return res?.content?.[0]?.text;
}
async chatComplete(options: ChatOptions): Promise<void> {
const ClaudeMapper: Record<RequestMessage["role"], string> = {
assistant: "Assistant",
user: "Human",
system: "Human",
};
async chat(options: ChatOptions): Promise<void> {
const visionModel = isVisionModel(options.config.model);

const accessStore = useAccessStore.getState();

const shouldStream = !!options.config.stream;

const prompt = options.messages
.map((v) => ({
role: ClaudeMapper[v.role] ?? "Human",
content: v.content,
}))
.map((v) => `\n\n${v.role}: ${v.content}`)
.join("");

const modelConfig = {
...useAppConfig.getState().modelConfig,
...useChatStore.getState().currentSession().mask.modelConfig,
@@ -102,142 +92,28 @@ export class ClaudeApi implements LLMApi {
},
};

const requestBody: ChatRequest = {
prompt,
stream: shouldStream,

model: modelConfig.model,
max_tokens_to_sample: modelConfig.max_tokens,
temperature: modelConfig.temperature,
top_p: modelConfig.top_p,
// top_k: modelConfig.top_k,
top_k: 5,
};

const path = this.path(Anthropic.ChatPath1);

const controller = new AbortController();
options.onController?.(controller);

const payload = {
method: "POST",
body: JSON.stringify(requestBody),
signal: controller.signal,
headers: {
"Content-Type": "application/json",
// Accept: "application/json",
"x-api-key": accessStore.anthropicApiKey,
"anthropic-version": accessStore.anthropicApiVersion,
Authorization: getAuthKey(accessStore.anthropicApiKey),
},
// mode: "no-cors" as RequestMode,
};

if (shouldStream) {
try {
const context = {
text: "",
finished: false,
};

const finish = () => {
if (!context.finished) {
options.onFinish(context.text);
context.finished = true;
}
};

controller.signal.onabort = finish;

fetchEventSource(path, {
...payload,
async onopen(res) {
const contentType = res.headers.get("content-type");
console.log("response content type: ", contentType);

if (contentType?.startsWith("text/plain")) {
context.text = await res.clone().text();
return finish();
}

if (
!res.ok ||
!res.headers
.get("content-type")
?.startsWith(EventStreamContentType) ||
res.status !== 200
) {
const responseTexts = [context.text];
let extraInfo = await res.clone().text();
try {
const resJson = await res.clone().json();
extraInfo = prettyObject(resJson);
} catch {}

if (res.status === 401) {
responseTexts.push(Locale.Error.Unauthorized);
}
const messages = [...options.messages];

if (extraInfo) {
responseTexts.push(extraInfo);
}
const keys = ["system", "user"];

context.text = responseTexts.join("\n\n");
// roles must alternate between "user" and "assistant" in claude, so add a fake assistant message between two user messages
for (let i = 0; i < messages.length - 1; i++) {
const message = messages[i];
const nextMessage = messages[i + 1];

return finish();
}
},
onmessage(msg) {
if (msg.data === "[DONE]" || context.finished) {
return finish();
}
const chunk = msg.data;
try {
const chunkJson = JSON.parse(chunk) as ChatStreamResponse;
const delta = chunkJson.completion;
if (delta) {
context.text += delta;
options.onUpdate?.(context.text, delta);
}
} catch (e) {
console.error("[Request] parse error", chunk, msg);
}
},
onclose() {
finish();
if (keys.includes(message.role) && keys.includes(nextMessage.role)) {
messages[i] = [
message,
{
role: "assistant",
content: ";",
},
onerror(e) {
options.onError?.(e);
},
openWhenHidden: true,
});
} catch (e) {
console.error("failed to chat", e);
options.onError?.(e as Error);
}
} else {
try {
controller.signal.onabort = () => options.onFinish("");

const res = await fetch(path, payload);
const resJson = await res.json();

const message = this.extractMessage(resJson);
options.onFinish(message);
} catch (e) {
console.error("failed to chat", e);
options.onError?.(e as Error);
] as any;
}
}
}
async chat(options: ChatOptions): Promise<void> {
const visionModel = isVisionModel(options.config.model);

const accessStore = useAccessStore.getState();

const shouldStream = !!options.config.stream;

const prompt = options.messages
const prompt = messages
.flat()
.filter((v) => {
if (!v.content) return false;
if (typeof v.content === "string" && !v.content.trim()) return false;
@@ -285,14 +161,6 @@ export class ClaudeApi implements LLMApi {
};
});

const modelConfig = {
...useAppConfig.getState().modelConfig,
...useChatStore.getState().currentSession().mask.modelConfig,
...{
model: options.config.model,
},
};

const requestBody: AnthropicChatRequest = {
messages: prompt,
stream: shouldStream,
1 change: 0 additions & 1 deletion app/store/chat.ts
Original file line number Diff line number Diff line change
@@ -496,7 +496,6 @@ export const useChatStore = createPersistStore(
tokenCount += estimateTokenLength(getMessageTextContent(msg));
reversedRecentMessages.push(msg);
}

// concat all messages
const recentMessages = [
...systemPrompts,