fix: local stream

neodyland · Jun 6, 2024 · 7235044 · 7235044
1 parent e1c2b38
commit 7235044
Showing 1 changed file with 11 additions and 5 deletions.
diff --git a/src/chat/llama.ts b/src/chat/llama.ts
@@ -40,11 +40,17 @@ async function* generate(chat: Chat[], system?: string) {
 			})),
 		),
 	});
-	const data: any = await res.body.json();
-	yield {
-		tokens: data.tokens,
-		content: data.content,
-	};
+	const stream = (await res.body.blob()).stream();
+	while (true) {
+		const { done, value } = await stream.getReader().read();
+		if (done) {
+			break;
+		}
+		yield {
+			tokens: 1,
+			content: value,
+		};
+	}
 }
 
 export const llama: ChatModel = {