diff --git a/apps/web/components/dashboard/admin/AdminActions.tsx b/apps/web/components/dashboard/admin/AdminActions.tsx index 3b95045c..02a2156a 100644 --- a/apps/web/components/dashboard/admin/AdminActions.tsx +++ b/apps/web/components/dashboard/admin/AdminActions.tsx @@ -69,6 +69,21 @@ export default function AdminActions() { }, }); + const { mutateAsync: reEmbedBookmarks, isPending: isReEmbedPending } = + api.admin.reEmbedAllBookmarks.useMutation({ + onSuccess: () => { + toast({ + description: "ReEmbed request has been enqueued!", + }); + }, + onError: (e) => { + toast({ + variant: "destructive", + description: e.message, + }); + }, + }); + return (
{t("common.actions")}
@@ -124,6 +139,13 @@ export default function AdminActions() { > {t("admin.actions.reindex_all_bookmarks")} + reEmbedBookmarks()} + > + {t("admin.actions.reembed_all_bookmarks")} + & { text: string }; + +export class EmbeddingsWorker { + static build() { + logger.info("Starting embeddings worker ..."); + const worker = new Runner( + EmbeddingsQueue, + { + run: runEmbeddings, + onComplete: async (job) => { + const jobId = job.id; + logger.info(`[embeddings][${jobId}] Completed successfully`); + return Promise.resolve(); + }, + onError: async (job) => { + const jobId = job.id; + logger.error( + `[embeddings][${jobId}] embeddings job failed: ${job.error}\n${job.error.stack}`, + ); + return Promise.resolve(); + }, + }, + { + concurrency: 1, + pollIntervalMs: 1000, + timeoutSecs: serverConfig.inference.jobTimeoutSec, + validator: zEmbeddingsRequestSchema, + }, + ); + + return worker; + } +} + +async function fetchBookmark(linkId: string) { + return await db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, linkId), + with: { + link: true, + text: true, + asset: true, + }, + }); +} + +async function chunkText(text: string): Promise { + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 100, + chunkOverlap: 0, + }); + const texts = await textSplitter.splitText(text); + return texts.map((t) => ({ + embeddingType: "content_chunk", + text: t, + fromOffset: 0, + toOffset: t.length, + })); +} + +async function prepareEmbeddings( + bookmark: NonNullable>>, +) { + const reqs: EmbeddingChunk[] = []; + + if (bookmark.link) { + if (bookmark.link.description) { + reqs.push({ + embeddingType: "description", + fromOffset: 0, + toOffset: bookmark.link.description?.length ?? 0, + text: bookmark.link.description ?? "", + }); + } + if (bookmark.link.content) { + reqs.push({ + embeddingType: "content_full", + fromOffset: 0, + toOffset: bookmark.link.content?.length ?? 0, + text: bookmark.link.content ?? "", + }); + reqs.push(...(await chunkText(bookmark.link.content ?? ""))); + } + } + + if (bookmark.text) { + if (bookmark.text.text) { + reqs.push({ + embeddingType: "description", + fromOffset: 0, + toOffset: bookmark.text.text?.length ?? 0, + text: bookmark.text.text ?? "", + }); + reqs.push(...(await chunkText(bookmark.text.text))); + } + } + + if (bookmark.asset) { + if (bookmark.asset.content) { + reqs.push({ + embeddingType: "content_full", + fromOffset: 0, + toOffset: bookmark.asset.content?.length ?? 0, + text: bookmark.asset.content ?? "", + }); + reqs.push(...(await chunkText(bookmark.asset.content))); + } + } + return reqs; +} + +async function runEmbeddings(job: DequeuedJob) { + const jobId = job.id; + + const inferenceClient = InferenceClientFactory.build(); + if (!inferenceClient) { + logger.debug( + `[embeddings][${jobId}] No inference client configured, nothing to do now`, + ); + return; + } + + const { bookmarkId } = job.data; + const bookmark = await fetchBookmark(bookmarkId); + if (!bookmark) { + throw new Error( + `[embeddings][${jobId}] bookmark with id ${bookmarkId} was not found`, + ); + } + + logger.info( + `[embeddings][${jobId}] Starting an embeddings job for bookmark with id "${bookmark.id}"`, + ); + + const reqs = await prepareEmbeddings(bookmark); + + logger.info(`[embeddings][${jobId}] Got ${reqs.length} embeddings requests`); + if (reqs.length == 0) { + logger.info(`[embeddings][${jobId}] No embeddings requests to process`); + return; + } + + const embeddings = await inferenceClient.generateEmbeddingFromText( + reqs.map((r) => r.text), + ); + + const resps = reqs.map((req, i) => ({ + ...req, + embedding: embeddings.embeddings[i], + })); + + const db = await getBookmarkVectorDb(); + // Delete the old vectors + await db.delete(`bookmarkid = "${bookmark.id}"`); + // Add the new vectors + await db.add( + resps.map((r) => ({ + vector: r.embedding, + bookmarkid: bookmarkId, + })), + ); +} diff --git a/apps/workers/index.ts b/apps/workers/index.ts index c2d3f28a..246f3d40 100644 --- a/apps/workers/index.ts +++ b/apps/workers/index.ts @@ -1,6 +1,7 @@ import "dotenv/config"; import { AssetPreprocessingWorker } from "assetPreprocessingWorker"; +import { EmbeddingsWorker } from "embeddingsWorker"; import { FeedRefreshingWorker, FeedWorker } from "feedWorker"; import { TidyAssetsWorker } from "tidyAssetsWorker"; @@ -18,16 +19,25 @@ async function main() { logger.info(`Workers version: ${serverConfig.serverVersion ?? "not set"}`); runQueueDBMigrations(); - const [crawler, openai, search, tidyAssets, video, feed, assetPreprocessing] = - [ - await CrawlerWorker.build(), - OpenAiWorker.build(), - SearchIndexingWorker.build(), - TidyAssetsWorker.build(), - VideoWorker.build(), - FeedWorker.build(), - AssetPreprocessingWorker.build(), - ]; + const [ + crawler, + openai, + search, + tidyAssets, + video, + feed, + assetPreprocessing, + embeddingsWorker, + ] = [ + await CrawlerWorker.build(), + OpenAiWorker.build(), + SearchIndexingWorker.build(), + TidyAssetsWorker.build(), + VideoWorker.build(), + FeedWorker.build(), + AssetPreprocessingWorker.build(), + EmbeddingsWorker.build(), + ]; FeedRefreshingWorker.start(); await Promise.any([ @@ -39,11 +49,12 @@ async function main() { video.run(), feed.run(), assetPreprocessing.run(), + embeddingsWorker.run(), ]), shutdownPromise, ]); logger.info( - "Shutting down crawler, openai, tidyAssets, video, feed, assetPreprocessing and search workers ...", + "Shutting down crawler, openai, tidyAssets, video, feed, assetPreprocessing, embeddingsWorker and search workers ...", ); FeedRefreshingWorker.stop(); @@ -54,6 +65,7 @@ async function main() { video.stop(); feed.stop(); assetPreprocessing.stop(); + embeddingsWorker.stop(); } main(); diff --git a/apps/workers/package.json b/apps/workers/package.json index 1ab2a934..837c8313 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -9,6 +9,8 @@ "@hoarder/shared": "workspace:^0.1.0", "@hoarder/trpc": "workspace:^0.1.0", "@hoarder/tsconfig": "workspace:^0.1.0", + "@langchain/core": "^0.3.26", + "@langchain/textsplitters": "^0.1.0", "@mozilla/readability": "^0.5.0", "@tsconfig/node21": "^21.0.1", "async-mutex": "^0.4.1", diff --git a/packages/db/drizzle/0037_sturdy_leper_queen.sql b/packages/db/drizzle/0037_sturdy_leper_queen.sql new file mode 100644 index 00000000..e3d187ea --- /dev/null +++ b/packages/db/drizzle/0037_sturdy_leper_queen.sql @@ -0,0 +1,14 @@ +CREATE TABLE `bookmarkEmbeddings` ( + `id` text PRIMARY KEY NOT NULL, + `bookmarkId` text NOT NULL, + `userId` text NOT NULL, + `embedding` text NOT NULL, + `embeddingType` text NOT NULL, + `fromOffset` integer, + `toOffset` integer, + FOREIGN KEY (`bookmarkId`) REFERENCES `bookmarks`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `bookmarkEmbeddings_bookmarkId_idx` ON `bookmarkEmbeddings` (`bookmarkId`);--> statement-breakpoint +CREATE INDEX `bookmarkEmbeddings_userId_idx` ON `bookmarkEmbeddings` (`userId`); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0037_snapshot.json b/packages/db/drizzle/meta/0037_snapshot.json new file mode 100644 index 00000000..7daadb67 --- /dev/null +++ b/packages/db/drizzle/meta/0037_snapshot.json @@ -0,0 +1,1627 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5de3851d-dbaf-45b4-8a8f-aee04ae0b53d", + "prevId": "7c51445d-0b54-4a42-aad3-630b85601478", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": [ + "provider", + "providerAccountId" + ], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {} + }, + "apiKey": { + "name": "apiKey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyId": { + "name": "keyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apiKey_keyId_unique": { + "name": "apiKey_keyId_unique", + "columns": [ + "keyId" + ], + "isUnique": true + }, + "apiKey_name_userId_unique": { + "name": "apiKey_name_userId_unique", + "columns": [ + "name", + "userId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "apiKey_userId_user_id_fk": { + "name": "apiKey_userId_user_id_fk", + "tableFrom": "apiKey", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "assets": { + "name": "assets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "contentType": { + "name": "contentType", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "assets_bookmarkId_idx": { + "name": "assets_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "assets_assetType_idx": { + "name": "assets_assetType_idx", + "columns": [ + "assetType" + ], + "isUnique": false + }, + "assets_userId_idx": { + "name": "assets_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "assets_bookmarkId_bookmarks_id_fk": { + "name": "assets_bookmarkId_bookmarks_id_fk", + "tableFrom": "assets", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assets_userId_user_id_fk": { + "name": "assets_userId_user_id_fk", + "tableFrom": "assets", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkAssets": { + "name": "bookmarkAssets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assetId": { + "name": "assetId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkAssets_id_bookmarks_id_fk": { + "name": "bookmarkAssets_id_bookmarks_id_fk", + "tableFrom": "bookmarkAssets", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkEmbeddings": { + "name": "bookmarkEmbeddings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embedding": { + "name": "embedding", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embeddingType": { + "name": "embeddingType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fromOffset": { + "name": "fromOffset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "toOffset": { + "name": "toOffset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarkEmbeddings_bookmarkId_idx": { + "name": "bookmarkEmbeddings_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "bookmarkEmbeddings_userId_idx": { + "name": "bookmarkEmbeddings_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkEmbeddings_bookmarkId_bookmarks_id_fk": { + "name": "bookmarkEmbeddings_bookmarkId_bookmarks_id_fk", + "tableFrom": "bookmarkEmbeddings", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarkEmbeddings_userId_user_id_fk": { + "name": "bookmarkEmbeddings_userId_user_id_fk", + "tableFrom": "bookmarkEmbeddings", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkLinks": { + "name": "bookmarkLinks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imageUrl": { + "name": "imageUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favicon": { + "name": "favicon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "htmlContent": { + "name": "htmlContent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawledAt": { + "name": "crawledAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawlStatus": { + "name": "crawlStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "crawlStatusCode": { + "name": "crawlStatusCode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 200 + } + }, + "indexes": { + "bookmarkLinks_url_idx": { + "name": "bookmarkLinks_url_idx", + "columns": [ + "url" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkLinks_id_bookmarks_id_fk": { + "name": "bookmarkLinks_id_bookmarks_id_fk", + "tableFrom": "bookmarkLinks", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkLists": { + "name": "bookmarkLists", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarkLists_userId_idx": { + "name": "bookmarkLists_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkLists_userId_user_id_fk": { + "name": "bookmarkLists_userId_user_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarkLists_parentId_bookmarkLists_id_fk": { + "name": "bookmarkLists_parentId_bookmarkLists_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "parentId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkTags": { + "name": "bookmarkTags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarkTags_name_idx": { + "name": "bookmarkTags_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "bookmarkTags_userId_idx": { + "name": "bookmarkTags_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkTags_userId_name_unique": { + "name": "bookmarkTags_userId_name_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkTags_userId_user_id_fk": { + "name": "bookmarkTags_userId_user_id_fk", + "tableFrom": "bookmarkTags", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkTexts": { + "name": "bookmarkTexts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkTexts_id_bookmarks_id_fk": { + "name": "bookmarkTexts_id_bookmarks_id_fk", + "tableFrom": "bookmarkTexts", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarks": { + "name": "bookmarks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "favourited": { + "name": "favourited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "taggingStatus": { + "name": "taggingStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarks_userId_idx": { + "name": "bookmarks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarks_archived_idx": { + "name": "bookmarks_archived_idx", + "columns": [ + "archived" + ], + "isUnique": false + }, + "bookmarks_favourited_idx": { + "name": "bookmarks_favourited_idx", + "columns": [ + "favourited" + ], + "isUnique": false + }, + "bookmarks_createdAt_idx": { + "name": "bookmarks_createdAt_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarks_userId_user_id_fk": { + "name": "bookmarks_userId_user_id_fk", + "tableFrom": "bookmarks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarksInLists": { + "name": "bookmarksInLists", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarksInLists_bookmarkId_idx": { + "name": "bookmarksInLists_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "bookmarksInLists_listId_idx": { + "name": "bookmarksInLists_listId_idx", + "columns": [ + "listId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarksInLists_bookmarkId_bookmarks_id_fk": { + "name": "bookmarksInLists_bookmarkId_bookmarks_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarksInLists_listId_bookmarkLists_id_fk": { + "name": "bookmarksInLists_listId_bookmarkLists_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "listId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bookmarksInLists_bookmarkId_listId_pk": { + "columns": [ + "bookmarkId", + "listId" + ], + "name": "bookmarksInLists_bookmarkId_listId_pk" + } + }, + "uniqueConstraints": {} + }, + "config": { + "name": "config", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "customPrompts": { + "name": "customPrompts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachedBy": { + "name": "attachedBy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "customPrompts_userId_idx": { + "name": "customPrompts_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "customPrompts_userId_user_id_fk": { + "name": "customPrompts_userId_user_id_fk", + "tableFrom": "customPrompts", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "highlights": { + "name": "highlights", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "startOffset": { + "name": "startOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endOffset": { + "name": "endOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'yellow'" + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "highlights_bookmarkId_idx": { + "name": "highlights_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "highlights_userId_idx": { + "name": "highlights_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "highlights_bookmarkId_bookmarks_id_fk": { + "name": "highlights_bookmarkId_bookmarks_id_fk", + "tableFrom": "highlights", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "highlights_userId_user_id_fk": { + "name": "highlights_userId_user_id_fk", + "tableFrom": "highlights", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "rssFeedImports": { + "name": "rssFeedImports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entryId": { + "name": "entryId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rssFeedId": { + "name": "rssFeedId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "rssFeedImports_feedIdIdx_idx": { + "name": "rssFeedImports_feedIdIdx_idx", + "columns": [ + "rssFeedId" + ], + "isUnique": false + }, + "rssFeedImports_entryIdIdx_idx": { + "name": "rssFeedImports_entryIdIdx_idx", + "columns": [ + "entryId" + ], + "isUnique": false + }, + "rssFeedImports_rssFeedId_entryId_unique": { + "name": "rssFeedImports_rssFeedId_entryId_unique", + "columns": [ + "rssFeedId", + "entryId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "rssFeedImports_rssFeedId_rssFeeds_id_fk": { + "name": "rssFeedImports_rssFeedId_rssFeeds_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "rssFeeds", + "columnsFrom": [ + "rssFeedId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "rssFeedImports_bookmarkId_bookmarks_id_fk": { + "name": "rssFeedImports_bookmarkId_bookmarks_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "rssFeeds": { + "name": "rssFeeds", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastFetchedAt": { + "name": "lastFetchedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastFetchedStatus": { + "name": "lastFetchedStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "rssFeeds_userId_idx": { + "name": "rssFeeds_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rssFeeds_userId_user_id_fk": { + "name": "rssFeeds_userId_user_id_fk", + "tableFrom": "rssFeeds", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "tagsOnBookmarks": { + "name": "tagsOnBookmarks", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachedAt": { + "name": "attachedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachedBy": { + "name": "attachedBy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tagsOnBookmarks_tagId_idx": { + "name": "tagsOnBookmarks_tagId_idx", + "columns": [ + "tagId" + ], + "isUnique": false + }, + "tagsOnBookmarks_bookmarkId_idx": { + "name": "tagsOnBookmarks_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tagsOnBookmarks_bookmarkId_bookmarks_id_fk": { + "name": "tagsOnBookmarks_bookmarkId_bookmarks_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tagsOnBookmarks_tagId_bookmarkTags_id_fk": { + "name": "tagsOnBookmarks_tagId_bookmarkTags_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "tagId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tagsOnBookmarks_bookmarkId_tagId_pk": { + "columns": [ + "bookmarkId", + "tagId" + ], + "name": "tagsOnBookmarks_bookmarkId_tagId_pk" + } + }, + "uniqueConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'user'" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": [ + "identifier", + "token" + ], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 7dfcd863..9d935b23 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -260,6 +260,13 @@ "when": 1735308236125, "tag": "0036_luxuriant_white_queen", "breakpoints": true + }, + { + "idx": 37, + "version": "6", + "when": 1735497866288, + "tag": "0037_sturdy_leper_queen", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/schema.ts b/packages/db/schema.ts index a8fe9eeb..3c691645 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -223,7 +223,9 @@ export const highlights = sqliteTable( endOffset: integer("endOffset").notNull(), color: text("color", { enum: ["red", "green", "blue", "yellow"], - }).default("yellow").notNull(), + }) + .default("yellow") + .notNull(), text: text("text"), note: text("note"), createdAt: createdAtField(), @@ -259,6 +261,32 @@ export const bookmarkAssets = sqliteTable("bookmarkAssets", { sourceUrl: text("sourceUrl"), }); +export const bookmarkEmbeddings = sqliteTable( + "bookmarkEmbeddings", + { + id: text("id") + .notNull() + .primaryKey() + .$defaultFn(() => createId()), + bookmarkId: text("bookmarkId") + .notNull() + .references(() => bookmarks.id, { onDelete: "cascade" }), + userId: text("userId") + .notNull() + .references(() => users.id, { onDelete: "cascade" }), + embedding: text("embedding").notNull(), + embeddingType: text("embeddingType", { + enum: ["description", "content_full", "content_chunk"], + }).notNull(), + fromOffset: integer("fromOffset"), + toOffset: integer("toOffset"), + }, + (tb) => ({ + bookmarkIdIdx: index("bookmarkEmbeddings_bookmarkId_idx").on(tb.bookmarkId), + userIdIdx: index("bookmarkEmbeddings_userId_idx").on(tb.userId), + }), +); + export const bookmarkTags = sqliteTable( "bookmarkTags", { diff --git a/packages/shared/package.json b/packages/shared/package.json index d741b70f..b0bdb6fd 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -5,6 +5,8 @@ "private": true, "type": "module", "dependencies": { + "@lancedb/lancedb": "^0.14.1", + "apache-arrow": "^18.1.0", "glob": "^11.0.0", "liteque": "^0.3.0", "meilisearch": "^0.37.0", diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 7afb8774..17dd7a1d 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -158,3 +158,19 @@ export const AssetPreprocessingQueue = keepFailedJobs: false, }, ); + +// Embeddings Queue +export const zEmbeddingsRequestSchema = z.object({ + bookmarkId: z.string(), +}); +export type EmbeddingsRequest = z.infer; +export const EmbeddingsQueue = new SqliteQueue( + "embeddings_queue", + queueDB, + { + defaultJobArgs: { + numRetries: 3, + }, + keepFailedJobs: false, + }, +); diff --git a/packages/shared/vectorDb.ts b/packages/shared/vectorDb.ts new file mode 100644 index 00000000..c28ce919 --- /dev/null +++ b/packages/shared/vectorDb.ts @@ -0,0 +1,25 @@ +import path from "path"; +import * as lancedb from "@lancedb/lancedb"; +import { Field, FixedSizeList, Float32, Schema, Utf8 } from "apache-arrow"; + +import serverConfig from "./config"; + +export async function getBookmarkVectorDb() { + const dbPath = path.join(serverConfig.dataDir, "vectordb"); + const db = await lancedb.connect(dbPath); + const table = db.createEmptyTable( + "bookmarks", + new Schema([ + new Field( + "vector", + new FixedSizeList(1536, new Field("item", new Float32(), true)), + ), + new Field("bookmarkid", new Utf8()), + ]), + { + mode: "create", + existOk: true, + }, + ); + return table; +} diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index c7dd7575..ddbf3abd 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { assets, bookmarkLinks, bookmarks, users } from "@hoarder/db/schema"; import serverConfig from "@hoarder/shared/config"; import { + EmbeddingsQueue, LinkCrawlerQueue, OpenAIQueue, SearchIndexingQueue, @@ -154,6 +155,17 @@ export const adminAppRouter = router({ await Promise.all(bookmarkIds.map((b) => triggerSearchReindex(b.id))); }), + reEmbedAllBookmarks: adminProcedure.mutation(async ({ ctx }) => { + const bookmarkIds = await ctx.db.query.bookmarks.findMany({ + columns: { + id: true, + }, + }); + + await Promise.all( + bookmarkIds.map((b) => EmbeddingsQueue.enqueue({ bookmarkId: b.id })), + ); + }), reRunInferenceOnAllBookmarks: adminProcedure .input( z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0df7d25d..6924aaba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -731,6 +731,12 @@ importers: '@hoarder/tsconfig': specifier: workspace:^0.1.0 version: link:../../tooling/typescript + '@langchain/core': + specifier: ^0.3.26 + version: 0.3.26 + '@langchain/textsplitters': + specifier: ^0.1.0 + version: 0.1.0(@langchain/core@0.3.26) '@mozilla/readability': specifier: ^0.5.0 version: 0.5.0 @@ -957,6 +963,12 @@ importers: packages/shared: dependencies: + '@lancedb/lancedb': + specifier: ^0.14.1 + version: 0.14.1(apache-arrow@18.1.0) + apache-arrow: + specifier: ^18.1.0 + version: 18.1.0 glob: specifier: ^11.0.0 version: 11.0.0 @@ -2301,6 +2313,9 @@ packages: resolution: {integrity: sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==} engines: {node: '>=6.9.0'} + '@cfworker/json-schema@4.0.3': + resolution: {integrity: sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==} + '@colors/colors@1.5.0': resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} @@ -3196,6 +3211,66 @@ packages: resolution: {integrity: sha512-BtwBDZjqZmhabWHfh4O/psDPbmcZotF2JEi40lRorViQkggaG0yuS8dy47LgKjEZN3P+wDQF7NE2iFBLf+oSFQ==} engines: {node: '>= 16'} + '@lancedb/lancedb-darwin-arm64@0.14.1': + resolution: {integrity: sha512-eSWV3GydXfyaptPXZ+S3BgXY1YI26oHQDekACaVevRW6/YQD7sS9UhhSZn1mYyDtLTfJu2kOK2XHA9UY8nyuTg==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [darwin] + + '@lancedb/lancedb-darwin-x64@0.14.1': + resolution: {integrity: sha512-ecf50ykF9WCWmpwAjs3Mk2mph7d+rMJ9EVJeX0UJ4KHDC874lnTDo6Tfd9iUcbExtNI1KZbu+CFnYsbQU+R0gw==} + engines: {node: '>= 18'} + cpu: [x64] + os: [darwin] + + '@lancedb/lancedb-linux-arm64-gnu@0.14.1': + resolution: {integrity: sha512-X7ub1fOm7jZ19KFW/u3nDyFvj5XzDPqEVrp9mmcOgSrst3NJEGGBz1JypkLnTWpg/7IpCBs1UO1G7R7LEsHYOA==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [linux] + + '@lancedb/lancedb-linux-arm64-musl@0.14.1': + resolution: {integrity: sha512-rkiWpsQCXwybwEjcdFXkAeGahiLcK/NQUjZc9WBY6CKk2Y9dICIafYzxZ6MDCY19jeJIgs3JS0mjleUWYr3JFw==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [linux] + + '@lancedb/lancedb-linux-x64-gnu@0.14.1': + resolution: {integrity: sha512-LGp4D58pQJ3+H3GncNxWHkvhIVOKpTzYUBtVfC8he1rwZ6+CiYDyK9Sim/j8o3UJlJ7cP0m3gNUzPfQchQF9WA==} + engines: {node: '>= 18'} + cpu: [x64] + os: [linux] + + '@lancedb/lancedb-linux-x64-musl@0.14.1': + resolution: {integrity: sha512-V/TeoyKUESPL/8L1z4WLbMFe5ZEv4gtxc0AFK8ghiduFYN/Hckuj4oTo/Y0ysLiBx1At9FCa91hWDB301ibHBg==} + engines: {node: '>= 18'} + cpu: [x64] + os: [linux] + + '@lancedb/lancedb-win32-x64-msvc@0.14.1': + resolution: {integrity: sha512-4M8D0j8/3WZv4CKo+Z44sISKPCKWN5MWA0dcEEGw4sEXHF2RJLrMIOOgEpT5NF7VW+X4t2JJxUA6j2T3cXaD8w==} + engines: {node: '>= 18'} + cpu: [x64] + os: [win32] + + '@lancedb/lancedb@0.14.1': + resolution: {integrity: sha512-DfJ887t52n/2s8G1JnzE7gAR4i7UnfP1OjDYnJ4yTk0aIcn76CbVOUegYfURYlYjL+QFdI1MrAzUdMgYgsGGcA==} + engines: {node: '>= 18'} + cpu: [x64, arm64] + os: [darwin, linux, win32] + peerDependencies: + apache-arrow: '>=15.0.0 <=18.1.0' + + '@langchain/core@0.3.26': + resolution: {integrity: sha512-6RUQHEp8wv+JwtYIIEBYBzbLlcAQZFc7EDOgAM0ukExjh9HiXoJzoWpgMRRCrr/koIbtwXPJUqBprZK1I1CXHQ==} + engines: {node: '>=18'} + + '@langchain/textsplitters@0.1.0': + resolution: {integrity: sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==} + engines: {node: '>=18'} + peerDependencies: + '@langchain/core': '>=0.2.21 <0.4.0' + '@leichtgewicht/ip-codec@2.0.4': resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} @@ -4485,6 +4560,9 @@ packages: '@swc/counter@0.1.3': resolution: {integrity: sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==} + '@swc/helpers@0.5.15': + resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==} + '@swc/helpers@0.5.5': resolution: {integrity: sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==} @@ -4602,6 +4680,12 @@ packages: '@types/chrome@0.0.260': resolution: {integrity: sha512-lX6QpgfsZRTDpNcCJ+3vzfFnFXq9bScFRTlfhbK5oecSAjamsno+ejFTCbNtc5O/TPnVK9Tja/PyecvWQe0F2w==} + '@types/command-line-args@5.2.3': + resolution: {integrity: sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==} + + '@types/command-line-usage@5.0.4': + resolution: {integrity: sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==} + '@types/connect-history-api-fallback@1.5.4': resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} @@ -4750,6 +4834,9 @@ packages: '@types/node@20.11.20': resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==} + '@types/node@20.17.10': + resolution: {integrity: sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==} + '@types/parse-json@4.0.2': resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} @@ -4843,6 +4930,9 @@ packages: '@types/unist@3.0.2': resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==} + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + '@types/ws@8.5.10': resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} @@ -5224,6 +5314,10 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + apache-arrow@18.1.0: + resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==} + hasBin: true + application-config-path@0.1.1: resolution: {integrity: sha512-zy9cHePtMP0YhwG+CfHm0bgwdnga2X3gZexpdCwEj//dpb+TKajtiC8REEUJUSq6Ab4f9cgNy2l8ObXzCXFkEw==} @@ -5255,6 +5349,14 @@ packages: resolution: {integrity: sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==} engines: {node: '>=0.10.0'} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@6.2.2: + resolution: {integrity: sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==} + engines: {node: '>=12.17'} + array-buffer-byte-length@1.0.1: resolution: {integrity: sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==} engines: {node: '>= 0.4'} @@ -5739,6 +5841,10 @@ packages: resolution: {integrity: sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==} engines: {node: '>=4'} + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + chalk@2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} @@ -5976,6 +6082,14 @@ packages: command-exists@1.2.9: resolution: {integrity: sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@7.0.3: + resolution: {integrity: sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==} + engines: {node: '>=12.20.0'} + commander@10.0.1: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} @@ -7491,6 +7605,10 @@ packages: resolution: {integrity: sha512-9ZonPT4ZAK4a+1pUPVPZJapbi7O5qbbJPdYw/NOQWZZbVLdDTYM3A4R9z/DpAM08IDaFGsvPgiGZ82WEwUDWjg==} engines: {node: '>=14.16'} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-root@1.1.0: resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} @@ -7518,6 +7636,9 @@ packages: resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} hasBin: true + flatbuffers@24.12.23: + resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==} + flatted@3.3.1: resolution: {integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==} @@ -8651,6 +8772,9 @@ packages: jose@5.2.2: resolution: {integrity: sha512-/WByRr4jDcsKlvMd1dRJnPfS1GVO3WuKyaurJ/vvXcOaUQO8rnNObCQMlv/5uCceVQIq5Q4WLF44ohsdiTohdg==} + js-tiktoken@1.0.16: + resolution: {integrity: sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -8716,6 +8840,10 @@ packages: engines: {node: '>=6'} hasBin: true + json-bignum@0.0.3: + resolution: {integrity: sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==} + engines: {node: '>=0.8'} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -8794,6 +8922,14 @@ packages: kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} + langsmith@0.2.14: + resolution: {integrity: sha512-ClAuAgSf3m9miMYotLEaZKQyKdaWlfjhebCuYco8bc6g72dU2VwTg31Bv4YINBq7EH2i1cMwbOiJxbOXPqjGig==} + peerDependencies: + openai: '*' + peerDependenciesMeta: + openai: + optional: true + language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} @@ -8954,6 +9090,9 @@ packages: resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.castarray@4.4.0: resolution: {integrity: sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==} @@ -9654,6 +9793,10 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mv@2.1.1: resolution: {integrity: sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==} engines: {node: '>=0.8.0'} @@ -10084,6 +10227,10 @@ packages: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + p-reflect@2.1.0: resolution: {integrity: sha512-paHV8NUz8zDHu5lhr/ngGWQiW067DK/+IbJ+RfZ4k+s8y4EKyYCz8pGYWjxCg35eHztpJAt+NUgvN4L+GCbPlg==} engines: {node: '>=8'} @@ -10092,6 +10239,10 @@ packages: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} @@ -11321,6 +11472,9 @@ packages: redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} + reflect-metadata@0.2.2: + resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} + reflect.getprototypeof@1.0.5: resolution: {integrity: sha512-62wgfC8dJWrmxv44CA36pLDnP6KKl3Vhxb7PL+8+qrrFMMoJij4vgiMP8zV4O8+CBMXY1mHxI5fITGHXFHVmQQ==} engines: {node: '>= 0.4'} @@ -12112,9 +12266,11 @@ packages: sudo-prompt@8.2.5: resolution: {integrity: sha512-rlBo3HU/1zAJUrkY6jNxDOC9eVYliG6nS4JA8u8KAshITd07tafMc/Br7xQwCSseXwJ2iCcHCE8SNWX3q8Z+kw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. sudo-prompt@9.1.1: resolution: {integrity: sha512-es33J1g2HjMpyAhz8lOR+ICmXXAqTuKbuXuUWLhOLew20oN9oUCgCJx615U/v7aioZg7IX5lIh9x34vwneu4pA==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} @@ -12160,6 +12316,10 @@ packages: resolution: {integrity: sha512-HwOKAP7Wc5aRGYdKH+dw0PRRpbO841v2DENBtjnR5HFWoiNByAl7vrx3p0G/rCyYXQsrxqtX48TImFtPcIHSpQ==} engines: {node: ^14.18.0 || >=16.0.0} + table-layout@4.1.1: + resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} + engines: {node: '>=12.17'} + table@6.8.2: resolution: {integrity: sha512-w2sfv80nrAh2VCbqR5AK27wswXhqcck2AhfnNW76beQXskGZ1V12GwS//yYVa3d3fcvAip2OUnbDAjW2k3v9fA==} engines: {node: '>=10.0.0'} @@ -12391,6 +12551,9 @@ packages: tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.7.1: resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==} engines: {node: '>=18.0.0'} @@ -12496,6 +12659,14 @@ packages: engines: {node: '>=14.17'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + ua-parser-js@1.0.37: resolution: {integrity: sha512-bhTyI94tZofjo+Dn8SN6Zv8nBDvyXTymAdM3LDI/0IboIUwTu1rEhW7v2TfiVsoYWgkQ4kOVqnI8APUFbIQIFQ==} @@ -12514,6 +12685,9 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.19.8: + resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + undici@6.19.8: resolution: {integrity: sha512-U8uCCl2x9TK3WANvmBavymRzxbfFYG+tAu+fgx3zxQy3qdagQqBLwJVrdyO1TBfUXvfKveMKJZhpvUYoOjM+4g==} engines: {node: '>=18.17'} @@ -12725,6 +12899,10 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + hasBin: true + uuid@3.4.0: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. @@ -13031,6 +13209,10 @@ packages: wonka@6.3.4: resolution: {integrity: sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==} + wordwrapjs@5.1.0: + resolution: {integrity: sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==} + engines: {node: '>=12.17'} + workbox-background-sync@6.6.0: resolution: {integrity: sha512-jkf4ZdgOJxC9u2vztxLuPT/UjlH7m/nWRQ/MgGL0v8BJHoZdVGJd18Kck+a0e55wGXdqyHO+4IQTk0685g4MUw==} @@ -13279,6 +13461,11 @@ packages: zlibjs@0.3.1: resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==} + zod-to-json-schema@3.24.1: + resolution: {integrity: sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==} + peerDependencies: + zod: ^3.24.1 + zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} @@ -14873,6 +15060,9 @@ snapshots: '@babel/helper-string-parser': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 + '@cfworker/json-schema@4.0.3': + dev: false + '@colors/colors@1.5.0': dev: false optional: true @@ -15528,7 +15718,7 @@ snapshots: '@docusaurus/theme-translations@3.5.2': dependencies: fs-extra: 11.2.0 - tslib: 2.6.2 + tslib: 2.8.1 dev: false '@docusaurus/tsconfig@3.5.2': @@ -15622,7 +15812,7 @@ snapshots: '@emnapi/runtime@1.1.1': dependencies: - tslib: 2.6.2 + tslib: 2.8.1 dev: false optional: true @@ -16051,7 +16241,7 @@ snapshots: password-prompt: 1.1.3 sudo-prompt: 8.2.5 tmp: 0.0.33 - tslib: 2.6.2 + tslib: 2.8.1 transitivePeerDependencies: - supports-color dev: false @@ -16435,7 +16625,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.11.20 + '@types/node': 20.17.10 jest-mock: 29.7.0 dev: false @@ -16443,7 +16633,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 20.11.20 + '@types/node': 20.17.10 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -16479,7 +16669,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.11.20 + '@types/node': 20.17.10 '@types/yargs': 17.0.32 chalk: 4.1.2 dev: false @@ -16533,6 +16723,72 @@ snapshots: mimic-fn: 3.0.0 dev: false + '@lancedb/lancedb-darwin-arm64@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-darwin-x64@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-linux-arm64-gnu@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-linux-arm64-musl@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-linux-x64-gnu@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-linux-x64-musl@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb-win32-x64-msvc@0.14.1': + dev: false + optional: true + + '@lancedb/lancedb@0.14.1(apache-arrow@18.1.0)': + dependencies: + apache-arrow: 18.1.0 + reflect-metadata: 0.2.2 + optionalDependencies: + '@lancedb/lancedb-darwin-arm64': 0.14.1 + '@lancedb/lancedb-darwin-x64': 0.14.1 + '@lancedb/lancedb-linux-arm64-gnu': 0.14.1 + '@lancedb/lancedb-linux-arm64-musl': 0.14.1 + '@lancedb/lancedb-linux-x64-gnu': 0.14.1 + '@lancedb/lancedb-linux-x64-musl': 0.14.1 + '@lancedb/lancedb-win32-x64-msvc': 0.14.1 + dev: false + + '@langchain/core@0.3.26': + dependencies: + '@cfworker/json-schema': 4.0.3 + ansi-styles: 5.2.0 + camelcase: 6.3.0 + decamelize: 1.2.0 + js-tiktoken: 1.0.16 + langsmith: 0.2.14 + mustache: 4.2.0 + p-queue: 6.6.2 + p-retry: 4.6.2 + uuid: 10.0.0 + zod: 3.22.4 + zod-to-json-schema: 3.24.1(zod@3.22.4) + transitivePeerDependencies: + - openai + dev: false + + '@langchain/textsplitters@0.1.0(@langchain/core@0.3.26)': + dependencies: + '@langchain/core': 0.3.26 + js-tiktoken: 1.0.16 + dev: false + '@leichtgewicht/ip-codec@2.0.4': dev: false @@ -18177,6 +18433,11 @@ snapshots: '@swc/counter@0.1.3': {} + '@swc/helpers@0.5.15': + dependencies: + tslib: 2.8.1 + dev: false + '@swc/helpers@0.5.5': dependencies: '@swc/counter': 0.1.3 @@ -18305,19 +18566,19 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/bonjour@3.5.13': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/cacheable-request@6.0.3': dependencies: '@types/http-cache-semantics': 4.0.4 '@types/keyv': 3.1.4 - '@types/node': 20.11.20 + '@types/node': 20.17.10 '@types/responselike': 1.0.3 dev: false @@ -18327,15 +18588,21 @@ snapshots: '@types/har-format': 1.2.15 dev: true + '@types/command-line-args@5.2.3': + dev: false + + '@types/command-line-usage@5.0.4': + dev: false + '@types/connect-history-api-fallback@1.5.4': dependencies: '@types/express-serve-static-core': 4.17.43 - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/connect@3.4.38': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/cookie@0.6.0': @@ -18381,7 +18648,7 @@ snapshots: '@types/express-serve-static-core@4.17.43': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 '@types/qs': 6.9.13 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -18406,12 +18673,12 @@ snapshots: '@types/glob@7.2.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/gtag.js@0.0.12': @@ -18451,7 +18718,7 @@ snapshots: '@types/http-proxy@1.17.14': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/istanbul-lib-coverage@2.0.6': @@ -18480,7 +18747,7 @@ snapshots: '@types/keyv@3.1.4': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/mdast@4.0.3': @@ -18514,7 +18781,7 @@ snapshots: '@types/node-forge@1.3.11': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/node@17.0.45': @@ -18529,6 +18796,10 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@20.17.10': + dependencies: + undici-types: 6.19.8 + '@types/parse-json@4.0.2': dev: false @@ -18600,12 +18871,12 @@ snapshots: '@types/resolve@1.17.1': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/responselike@1.0.3': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/retry@0.12.0': @@ -18613,7 +18884,7 @@ snapshots: '@types/sax@1.2.7': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/scheduler@0.16.8': {} @@ -18623,7 +18894,7 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/serve-index@1.9.4': @@ -18635,12 +18906,12 @@ snapshots: dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/sockjs@0.3.36': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/stack-utils@2.0.3': @@ -18658,9 +18929,12 @@ snapshots: '@types/unist@3.0.2': {} + '@types/uuid@10.0.0': + dev: false + '@types/ws@8.5.10': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false '@types/yargs-parser@21.0.3': @@ -18673,7 +18947,7 @@ snapshots: '@types/yauzl@2.10.3': dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 dev: false optional: true @@ -19197,6 +19471,19 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + apache-arrow@18.1.0: + dependencies: + '@swc/helpers': 0.5.15 + '@types/command-line-args': 5.2.3 + '@types/command-line-usage': 5.0.4 + '@types/node': 20.17.10 + command-line-args: 5.2.1 + command-line-usage: 7.0.3 + flatbuffers: 24.12.23 + json-bignum: 0.0.3 + tslib: 2.8.1 + dev: false + application-config-path@0.1.1: dev: false @@ -19233,6 +19520,12 @@ snapshots: arr-union@3.1.0: dev: false + array-back@3.1.0: + dev: false + + array-back@6.2.2: + dev: false + array-buffer-byte-length@1.0.1: dependencies: call-bind: 1.0.7 @@ -19322,12 +19615,12 @@ snapshots: ast-types@0.13.4: dependencies: - tslib: 2.6.2 + tslib: 2.8.1 dev: false ast-types@0.15.2: dependencies: - tslib: 2.6.2 + tslib: 2.8.1 dev: false astral-regex@2.0.0: @@ -19895,7 +20188,7 @@ snapshots: camel-case@4.1.2: dependencies: pascal-case: 3.1.2 - tslib: 2.6.2 + tslib: 2.8.1 dev: false camelcase-css@2.0.1: {} @@ -19948,6 +20241,11 @@ snapshots: type-detect: 4.0.8 dev: true + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + dev: false + chalk@2.4.2: dependencies: ansi-styles: 3.2.1 @@ -20047,7 +20345,7 @@ snapshots: chrome-launcher@0.15.2: dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 escape-string-regexp: 4.0.0 is-wsl: 2.2.0 lighthouse-logger: 1.4.2 @@ -20066,7 +20364,7 @@ snapshots: chromium-edge-launcher@0.2.0: dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 escape-string-regexp: 4.0.0 is-wsl: 2.2.0 lighthouse-logger: 1.4.2 @@ -20258,6 +20556,22 @@ snapshots: command-exists@1.2.9: dev: false + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + dev: false + + command-line-usage@7.0.3: + dependencies: + array-back: 6.2.2 + chalk-template: 0.4.0 + table-layout: 4.1.1 + typical: 7.3.0 + dev: false + commander@10.0.1: dev: false @@ -21105,7 +21419,7 @@ snapshots: dot-case@3.0.4: dependencies: no-case: 3.0.4 - tslib: 2.6.2 + tslib: 2.8.1 dev: false dot-prop@6.0.1: @@ -22360,6 +22674,11 @@ snapshots: pkg-dir: 7.0.0 dev: false + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + dev: false + find-root@1.1.0: dev: false @@ -22393,6 +22712,9 @@ snapshots: flat@5.0.2: {} + flatbuffers@24.12.23: + dev: false + flatted@3.3.1: {} flow-enums-runtime@0.0.6: @@ -23756,7 +24078,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 20.11.20 + '@types/node': 20.17.10 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -23785,7 +24107,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.11.20 + '@types/node': 20.17.10 jest-util: 29.7.0 dev: false @@ -23795,7 +24117,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.11.20 + '@types/node': 20.17.10 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -23814,20 +24136,20 @@ snapshots: jest-worker@26.6.2: dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 merge-stream: 2.0.0 supports-color: 7.2.0 dev: false jest-worker@27.5.1: dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 merge-stream: 2.0.0 supports-color: 8.1.1 jest-worker@29.7.0: dependencies: - '@types/node': 20.11.20 + '@types/node': 20.17.10 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -23855,6 +24177,11 @@ snapshots: jose@5.2.2: dev: false + js-tiktoken@1.0.16: + dependencies: + base64-js: 1.5.1 + dev: false + js-tokens@4.0.0: {} js-tokens@8.0.3: @@ -23977,6 +24304,9 @@ snapshots: jsesc@3.0.2: {} + json-bignum@0.0.3: + dev: false + json-buffer@3.0.1: {} json-parse-better-errors@1.0.2: @@ -24065,6 +24395,16 @@ snapshots: kuler@2.0.0: dev: false + langsmith@0.2.14: + dependencies: + '@types/uuid': 10.0.0 + commander: 10.0.1 + p-queue: 6.6.2 + p-retry: 4.6.2 + semver: 7.6.3 + uuid: 10.0.0 + dev: false + language-subtag-registry@0.3.22: dev: false @@ -24258,6 +24598,9 @@ snapshots: p-locate: 6.0.0 dev: false + lodash.camelcase@4.3.0: + dev: false + lodash.castarray@4.4.0: {} lodash.clonedeep@4.5.0: @@ -24328,7 +24671,7 @@ snapshots: lower-case@2.0.2: dependencies: - tslib: 2.6.2 + tslib: 2.8.1 dev: false lowercase-keys@2.0.0: @@ -25533,6 +25876,9 @@ snapshots: thunky: 1.1.0 dev: false + mustache@4.2.0: + dev: false + mv@2.1.1: dependencies: mkdirp: 0.5.6 @@ -25695,7 +26041,7 @@ snapshots: no-case@3.0.4: dependencies: lower-case: 2.0.2 - tslib: 2.6.2 + tslib: 2.8.1 dev: false node-abi@3.56.0: @@ -26103,6 +26449,12 @@ snapshots: aggregate-error: 3.1.0 dev: false + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + dev: false + p-reflect@2.1.0: dev: false @@ -26112,6 +26464,11 @@ snapshots: retry: 0.13.1 dev: false + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + dev: false + p-try@2.2.0: dev: false @@ -26148,7 +26505,7 @@ snapshots: param-case@3.0.4: dependencies: dot-case: 3.0.4 - tslib: 2.6.2 + tslib: 2.8.1 dev: false parent-module@1.0.1: @@ -26223,7 +26580,7 @@ snapshots: pascal-case@3.1.2: dependencies: no-case: 3.0.4 - tslib: 2.6.2 + tslib: 2.8.1 dev: false password-prompt@1.1.3: @@ -27545,7 +27902,7 @@ snapshots: '@types/react': 18.2.58 react: 18.3.1 react-style-singleton: 2.2.1(@types/react@18.2.58)(react@18.3.1) - tslib: 2.6.2 + tslib: 2.8.1 dev: false react-remove-scroll@2.5.5(@types/react@18.2.58)(react@18.3.1): @@ -27630,7 +27987,7 @@ snapshots: get-nonce: 1.0.1 invariant: 2.2.4 react: 18.3.1 - tslib: 2.6.2 + tslib: 2.8.1 dev: false react-syntax-highlighter@15.5.0(react@18.3.1): @@ -27694,7 +28051,7 @@ snapshots: ast-types: 0.15.2 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.6.2 + tslib: 2.8.1 dev: false rechoir@0.6.2: @@ -27722,6 +28079,9 @@ snapshots: '@babel/runtime': 7.26.0 dev: false + reflect-metadata@0.2.2: + dev: false + reflect.getprototypeof@1.0.5: dependencies: call-bind: 1.0.7 @@ -28511,7 +28871,7 @@ snapshots: snake-case@3.0.4: dependencies: dot-case: 3.0.4 - tslib: 2.6.2 + tslib: 2.8.1 dev: false sockjs@0.3.24: @@ -28889,9 +29249,15 @@ snapshots: synckit@0.8.8: dependencies: '@pkgr/core': 0.1.1 - tslib: 2.6.2 + tslib: 2.8.1 dev: true + table-layout@4.1.1: + dependencies: + array-back: 6.2.2 + wordwrapjs: 5.1.0 + dev: false + table@6.8.2: dependencies: ajv: 8.12.0 @@ -29201,6 +29567,8 @@ snapshots: tslib@2.6.2: {} + tslib@2.8.1: {} + tsx@4.7.1: dependencies: esbuild: 0.19.12 @@ -29318,6 +29686,12 @@ snapshots: typescript@5.3.3: {} + typical@4.0.0: + dev: false + + typical@7.3.0: + dev: false + ua-parser-js@1.0.37: dev: false @@ -29342,6 +29716,8 @@ snapshots: undici-types@5.26.5: {} + undici-types@6.19.8: {} + undici@6.19.8: {} unicode-canonical-property-names-ecmascript@2.0.0: @@ -29517,7 +29893,7 @@ snapshots: dependencies: '@types/react': 18.2.58 react: 18.3.1 - tslib: 2.6.2 + tslib: 2.8.1 dev: false use-chrome-storage@1.3.0(react@18.3.1): @@ -29546,7 +29922,7 @@ snapshots: '@types/react': 18.2.58 detect-node-es: 1.1.0 react: 18.3.1 - tslib: 2.6.2 + tslib: 2.8.1 dev: false use-sync-external-store@1.2.0(react@18.3.1): @@ -29583,6 +29959,9 @@ snapshots: utils-merge@1.0.1: dev: false + uuid@10.0.0: + dev: false + uuid@3.4.0: dev: false @@ -30031,6 +30410,9 @@ snapshots: wonka@6.3.4: dev: false + wordwrapjs@5.1.0: + dev: false + workbox-background-sync@6.6.0: dependencies: idb: 7.1.1 @@ -30353,6 +30735,11 @@ snapshots: zlibjs@0.3.1: dev: false + zod-to-json-schema@3.24.1(zod@3.22.4): + dependencies: + zod: 3.22.4 + dev: false + zod@3.22.4: dev: false