fix

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
 .env
 dist/
-bun.lockb
+bun.lockb
+tmp/

package.json

@@ -11,12 +11,10 @@
 	"dependencies": {
 		"@google/generative-ai": "^0.21.0",
 		"discord.js": "^14.16.3",
-		"dotenv": "^16.4.7",
-		"stream-json": "^1.9.1"
+		"dotenv": "^16.4.7"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^1.9.4",
-		"@types/stream-json": "^1.7.8",
 		"typescript": "^5.7.2"
 	}
 }

src/chat/gemini.ts

@@ -1,60 +1,51 @@
 import type { Chat, ChatModel } from ".";
 import { evar } from "../var";
-import { parser } from "stream-json";
-import { GoogleGenerativeAI } from "@google/generative-ai";
+import {
+	GoogleGenerativeAI,
+	HarmBlockThreshold,
+	HarmCategory,
+} from "@google/generative-ai";
+import { GoogleAIFileManager } from "@google/generative-ai/server";
 
 const geminiKey = evar("GEMINI_KEY");
 
 const ai = new GoogleGenerativeAI(geminiKey);
+export const files = new GoogleAIFileManager(geminiKey);
 
 async function* generateGeminiContent(
 	chat: Chat[],
 	model: string,
 	system?: string,
 ) {
-	const payload = {
-		safetySettings: [
-			"HARM_CATEGORY_SEXUALLY_EXPLICIT",
-			"HARM_CATEGORY_DANGEROUS_CONTENT",
-			"HARM_CATEGORY_HATE_SPEECH",
-			"HARM_CATEGORY_HARASSMENT",
-		].map((category) => ({
-			category,
-			threshold: "BLOCK_NONE",
-		})),
-		contents: chat.map((c) => ({
-			role: c.role === "user" ? "user" : "model",
-			parts: [
-				{
-					text: c.text,
-				},
-				...(c.attachment
-					? [
-							{
-								inlineData: {
-									mimeType: c.attachment.mime,
-									data: c.attachment.data,
-								},
-							},
-						]
-					: []),
-			],
-		})),
-		systemInstruction: system
-			? {
-					role: "model",
+	try {
+		const res = await ai
+			.getGenerativeModel({ model: model })
+			.generateContentStream({
+				safetySettings: [
+					HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+					HarmCategory.HARM_CATEGORY_HARASSMENT,
+					HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+					HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+				].map((category) => ({
+					category,
+					threshold: HarmBlockThreshold.BLOCK_NONE,
+				})),
+				systemInstruction: system,
+				contents: chat.map((c) => ({
+					role: c.role === "user" ? "user" : "model",
 					parts: [
 						{
-							text: system,
+							text: c.text,
 						},
+						...(c.attachment?.map((a) => ({
+							fileData: {
+								mimeType: a.mime,
+								fileUri: a.data,
+							},
+						})) || []),
 					],
-				}
-			: undefined,
-	};
-	try {
-		const res = await ai
-			.getGenerativeModel({ model: model })
-			.generateContentStream(chat);
+				})),
+			});
 		for await (const chunk of res.stream) {
 			yield {
 				tokens: 0,

src/chat/index.ts

@@ -3,15 +3,18 @@ import {
 	gemini15FlashSmall,
 	gemini15Pro,
 	gemini20Flash,
+	files,
 } from "./gemini";
+import { writeFile, rm, mkdir, stat } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
 
 export interface Chat {
 	role: "user" | "assistant" | "system";
 	text: string;
 	attachment?: {
 		mime: string;
 		data: string;
-	}; // base64
+	}[]; // base64
 }
 
 export interface ChatModel {
@@ -35,9 +38,18 @@ export const models = [
 	gemini20Flash,
 ];
 
-export async function getAttachmentBase64(url: string) {
-	const res = await fetch(url);
-	const data = await res.blob();
-	const buf = Buffer.from(await data.arrayBuffer());
-	return buf.toString("base64");
+export async function uploadAttachment(url: string, mime: string) {
+	const buf = await (await fetch(url)).arrayBuffer();
+	try {
+		(await stat("./tmp")).isDirectory();
+	} catch {
+		await mkdir("./tmp");
+	}
+	const path = `./tmp/${randomUUID()}`;
+	await writeFile(path, Buffer.from(buf));
+	try {
+		return (await files.uploadFile(path, { mimeType: mime })).file.uri;
+	} finally {
+		await rm(path);
+	}
 }

src/commands/ask.ts

@@ -4,7 +4,7 @@ import {
 	SlashCommandBuilder,
 } from "discord.js";
 import { Command } from ".";
-import { getAttachmentBase64, models } from "../chat";
+import { uploadAttachment, models } from "../chat";
 
 export const ask: Command = {
 	builder: new SlashCommandBuilder()
@@ -62,10 +62,15 @@ export const ask: Command = {
 				role: "user" as const,
 				text: question,
 				attachment: attachment
-					? {
-							mime: attachment.contentType!,
-							data: await getAttachmentBase64(attachment.url),
-						}
+					? [
+							{
+								mime: attachment.contentType || "application/octet-stream",
+								data: await uploadAttachment(
+									attachment.url,
+									attachment.contentType || "application/octet-stream",
+								),
+							},
+						]
 					: undefined,
 			},
 		];

src/index.ts

@@ -8,7 +8,7 @@ import {
 import { evar } from "./var";
 import { commands } from "./commands";
 import { addChatQueue } from "./queue";
-import { getAttachmentBase64 } from "./chat";
+import { uploadAttachment } from "./chat";
 
 export const client = new Client({
 	intents:
@@ -46,7 +46,10 @@ client.once(Events.ClientReady, async () => {
 const resolveAttachment = async (attachment: Attachment) => {
 	return {
 		mime: attachment.contentType!,
-		data: await getAttachmentBase64(attachment.url),
+		data: await uploadAttachment(
+			attachment.url,
+			attachment.contentType || "application/octet-stream",
+		),
 	};
 };
 
@@ -71,10 +74,9 @@ client.on(Events.MessageCreate, async (message) => {
 		{
 			text: message.content,
 			role: "user",
-			attachment:
-				message.attachments.size > 0
-					? await resolveAttachment(message.attachments.first()!)
-					: undefined,
+			attachment: await Promise.all(
+				message.attachments.map((a) => resolveAttachment(a)),
+			),
 			id: message.id,
 		},
 		model,