From 341fc5e1305304895873a8dcfdec34818cc9c390 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Wed, 30 Aug 2023 16:37:43 +0300 Subject: [PATCH] feat(nx-dev): use the vercel api and stream the results --- nx-dev/data-access-ai/.eslintrc.json | 18 - nx-dev/data-access-ai/README.md | 11 - nx-dev/data-access-ai/jest.config.ts | 11 - nx-dev/data-access-ai/package.json | 5 - nx-dev/data-access-ai/project.json | 30 -- nx-dev/data-access-ai/src/index.ts | 2 - .../data-access-ai/src/lib/data-access-ai.ts | 59 --- nx-dev/data-access-ai/src/lib/utils.ts | 42 -- nx-dev/data-access-ai/tsconfig.json | 24 -- nx-dev/data-access-ai/tsconfig.lib.json | 10 - nx-dev/data-access-ai/tsconfig.spec.json | 14 - nx-dev/feature-ai/src/lib/error-message.tsx | 6 + nx-dev/feature-ai/src/lib/feed-container.tsx | 116 ++--- .../feature-ai/src/lib/feed/feed-answer.tsx | 11 +- nx-dev/feature-ai/src/lib/feed/feed.tsx | 2 +- nx-dev/feature-ai/src/lib/prompt.tsx | 26 +- nx-dev/feature-ai/src/lib/utils.ts | 7 - nx-dev/nx-dev/pages/api/query-ai-handler.ts | 153 +++---- nx-dev/util-ai/package.json | 2 +- nx-dev/util-ai/src/index.ts | 1 - nx-dev/util-ai/src/lib/chat-utils.ts | 150 +++---- nx-dev/util-ai/src/lib/moderation.ts | 32 +- nx-dev/util-ai/src/lib/openai-call.ts | 26 -- nx-dev/util-ai/src/lib/utils.ts | 4 +- package.json | 10 +- pnpm-lock.yaml | 403 ++++++++++++++++-- .../create-embeddings/src/main.mts | 19 +- tsconfig.base.json | 1 - 28 files changed, 591 insertions(+), 604 deletions(-) delete mode 100644 nx-dev/data-access-ai/.eslintrc.json delete mode 100644 nx-dev/data-access-ai/README.md delete mode 100644 nx-dev/data-access-ai/jest.config.ts delete mode 100644 nx-dev/data-access-ai/package.json delete mode 100644 nx-dev/data-access-ai/project.json delete mode 100644 nx-dev/data-access-ai/src/index.ts delete mode 100644 nx-dev/data-access-ai/src/lib/data-access-ai.ts delete mode 100644 nx-dev/data-access-ai/src/lib/utils.ts delete mode 100644 nx-dev/data-access-ai/tsconfig.json delete mode 100644 nx-dev/data-access-ai/tsconfig.lib.json delete mode 100644 nx-dev/data-access-ai/tsconfig.spec.json delete mode 100644 nx-dev/feature-ai/src/lib/utils.ts delete mode 100644 nx-dev/util-ai/src/lib/openai-call.ts diff --git a/nx-dev/data-access-ai/.eslintrc.json b/nx-dev/data-access-ai/.eslintrc.json deleted file mode 100644 index 9d9c0db55bb1e9..00000000000000 --- a/nx-dev/data-access-ai/.eslintrc.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "extends": ["../../.eslintrc.json"], - "ignorePatterns": ["!**/*"], - "overrides": [ - { - "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], - "rules": {} - }, - { - "files": ["*.ts", "*.tsx"], - "rules": {} - }, - { - "files": ["*.js", "*.jsx"], - "rules": {} - } - ] -} diff --git a/nx-dev/data-access-ai/README.md b/nx-dev/data-access-ai/README.md deleted file mode 100644 index bbd268a800a01c..00000000000000 --- a/nx-dev/data-access-ai/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# nx-dev-data-access-ai - -This library was generated with [Nx](https://nx.dev). - -## Building - -Run `nx build nx-dev-data-access-ai` to build the library. - -## Running unit tests - -Run `nx test nx-dev-data-access-ai` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/nx-dev/data-access-ai/jest.config.ts b/nx-dev/data-access-ai/jest.config.ts deleted file mode 100644 index 1a4f1efd3bc641..00000000000000 --- a/nx-dev/data-access-ai/jest.config.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* eslint-disable */ -export default { - displayName: 'nx-dev-data-access-ai', - preset: '../../jest.preset.js', - testEnvironment: 'node', - transform: { - '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], - }, - moduleFileExtensions: ['ts', 'js', 'html'], - coverageDirectory: '../../coverage/nx-dev/data-access-ai', -}; diff --git a/nx-dev/data-access-ai/package.json b/nx-dev/data-access-ai/package.json deleted file mode 100644 index 4952715963dbb1..00000000000000 --- a/nx-dev/data-access-ai/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "@nx/nx-dev/data-access-ai", - "version": "0.0.1", - "type": "commonjs" -} diff --git a/nx-dev/data-access-ai/project.json b/nx-dev/data-access-ai/project.json deleted file mode 100644 index 346c09e383fda3..00000000000000 --- a/nx-dev/data-access-ai/project.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "nx-dev-data-access-ai", - "$schema": "../../node_modules/nx/schemas/project-schema.json", - "sourceRoot": "nx-dev/data-access-ai/src", - "projectType": "library", - "targets": { - "lint": { - "executor": "@nx/linter:eslint", - "outputs": ["{options.outputFile}"], - "options": { - "lintFilePatterns": ["nx-dev/data-access-ai/**/*.ts"] - } - }, - "test": { - "executor": "@nx/jest:jest", - "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], - "options": { - "jestConfig": "nx-dev/data-access-ai/jest.config.ts", - "passWithNoTests": true - }, - "configurations": { - "ci": { - "ci": true, - "codeCoverage": true - } - } - } - }, - "tags": [] -} diff --git a/nx-dev/data-access-ai/src/index.ts b/nx-dev/data-access-ai/src/index.ts deleted file mode 100644 index 4149f0e243fa5e..00000000000000 --- a/nx-dev/data-access-ai/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './lib/data-access-ai'; -export * from './lib/utils'; diff --git a/nx-dev/data-access-ai/src/lib/data-access-ai.ts b/nx-dev/data-access-ai/src/lib/data-access-ai.ts deleted file mode 100644 index 0e9ce7fccb33ea..00000000000000 --- a/nx-dev/data-access-ai/src/lib/data-access-ai.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { CreateCompletionResponseUsage } from 'openai'; -import { MAX_HISTORY_LENGTH, ChatItem } from '@nx/nx-dev/util-ai'; -import { getChatResponse } from './utils'; - -let chatFullHistory: ChatItem[] = []; - -let totalTokensSoFar = 0; - -export async function queryAi( - query: string, - aiResponse?: string -): Promise<{ - textResponse: string; - usage?: CreateCompletionResponseUsage; - sources: { heading: string; url: string }[]; - sourcesMarkdown: string; -}> { - if (chatFullHistory.length > MAX_HISTORY_LENGTH) { - chatFullHistory.slice(0, MAX_HISTORY_LENGTH - 4); - } - - try { - const responseObj = await getChatResponse( - query, - chatFullHistory, - aiResponse - ); - - if (!responseObj.ok) { - throw await responseObj.json(); - } - - const response: { - textResponse: string; - usage?: CreateCompletionResponseUsage; - sources: { heading: string; url: string }[]; - sourcesMarkdown: string; - chatHistory: ChatItem[]; - requestTokens: number; - } = await responseObj.json(); - - totalTokensSoFar += response.requestTokens; - chatFullHistory = response.chatHistory; - - return response; - } catch (e: any) { - console.error('Error: ', e?.['message'] || e); - throw e; - } -} - -export function resetHistory() { - chatFullHistory = []; - totalTokensSoFar = 0; -} - -export function getHistory(): ChatItem[] { - return chatFullHistory; -} diff --git a/nx-dev/data-access-ai/src/lib/utils.ts b/nx-dev/data-access-ai/src/lib/utils.ts deleted file mode 100644 index caa7516b7c5f92..00000000000000 --- a/nx-dev/data-access-ai/src/lib/utils.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { ChatCompletionRequestMessageRoleEnum } from 'openai'; -import { getHistory } from './data-access-ai'; -import { ChatItem } from '@nx/nx-dev/util-ai'; - -export function extractQuery(text: string) { - const regex = /---- My message: (.+)/; - const match = text.match(regex); - return match ? match[1].trim() : text; -} - -export function getProcessedHistory(): ChatItem[] { - let history = getHistory(); - history = history - .map((item) => { - if (item.role === ChatCompletionRequestMessageRoleEnum.User) { - item.content = extractQuery(item.content); - } - if (item.role !== ChatCompletionRequestMessageRoleEnum.System) { - return item; - } else { - return undefined; - } - }) - .filter((item) => !!item) as ChatItem[]; - return history; -} - -export function getChatResponse( - query: string, - chatFullHistory: ChatItem[], - aiResponse?: string -): Promise { - return fetch('/api/query-ai-handler', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - query, - chatFullHistory, - aiResponse, - }), - }); -} diff --git a/nx-dev/data-access-ai/tsconfig.json b/nx-dev/data-access-ai/tsconfig.json deleted file mode 100644 index dfe3c12605c555..00000000000000 --- a/nx-dev/data-access-ai/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "module": "commonjs", - "forceConsistentCasingInFileNames": true, - "strict": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "target": "es2021", - "lib": ["es2021", "DOM"] - }, - "files": [], - "include": [], - "references": [ - { - "path": "./tsconfig.lib.json" - }, - { - "path": "./tsconfig.spec.json" - } - ] -} diff --git a/nx-dev/data-access-ai/tsconfig.lib.json b/nx-dev/data-access-ai/tsconfig.lib.json deleted file mode 100644 index 33eca2c2cdf8c6..00000000000000 --- a/nx-dev/data-access-ai/tsconfig.lib.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "../../dist/out-tsc", - "declaration": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"], - "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] -} diff --git a/nx-dev/data-access-ai/tsconfig.spec.json b/nx-dev/data-access-ai/tsconfig.spec.json deleted file mode 100644 index 9b2a121d114b68..00000000000000 --- a/nx-dev/data-access-ai/tsconfig.spec.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "../../dist/out-tsc", - "module": "commonjs", - "types": ["jest", "node"] - }, - "include": [ - "jest.config.ts", - "src/**/*.test.ts", - "src/**/*.spec.ts", - "src/**/*.d.ts" - ] -} diff --git a/nx-dev/feature-ai/src/lib/error-message.tsx b/nx-dev/feature-ai/src/lib/error-message.tsx index 9e46d6aeef3adc..dd5e35234f655a 100644 --- a/nx-dev/feature-ai/src/lib/error-message.tsx +++ b/nx-dev/feature-ai/src/lib/error-message.tsx @@ -4,6 +4,12 @@ import { } from '@heroicons/react/24/outline'; export function ErrorMessage({ error }: { error: any }): JSX.Element { + try { + if (error.message?.includes('no_results')) { + error = JSON.parse(error.message); + } + } catch (e) {} + if (error?.data?.no_results) { return (
diff --git a/nx-dev/feature-ai/src/lib/feed-container.tsx b/nx-dev/feature-ai/src/lib/feed-container.tsx index 2aa1dfe9cfb662..bb95da8908acc1 100644 --- a/nx-dev/feature-ai/src/lib/feed-container.tsx +++ b/nx-dev/feature-ai/src/lib/feed-container.tsx @@ -1,22 +1,11 @@ -import { getProcessedHistory, queryAi } from '@nx/nx-dev/data-access-ai'; import { sendCustomEvent } from '@nx/nx-dev/feature-analytics'; import { RefObject, useEffect, useRef, useState } from 'react'; import { ErrorMessage } from './error-message'; import { Feed } from './feed/feed'; import { LoadingState } from './loading-state'; import { Prompt } from './prompt'; -import { formatMarkdownSources } from './utils'; -import { ChatItem } from '@nx/nx-dev/util-ai'; - -interface LastQueryMetadata { - sources: string[]; - textResponse: string; - usage: { - completion_tokens: number; - prompt_tokens: number; - total_tokens: number; - } | null; -} +import { ChatItem, extractLinksFromSourcesSection } from '@nx/nx-dev/util-ai'; +import { Message, useChat } from 'ai/react'; const assistantWelcome: ChatItem = { role: 'assistant', @@ -25,13 +14,30 @@ const assistantWelcome: ChatItem = { }; export function FeedContainer(): JSX.Element { - const [chatHistory, setChatHistory] = useState([]); - const [queryError, setQueryError] = useState(null); - const [isLoading, setIsLoading] = useState(false); - const [lastQueryMetadata, setLastQueryMetadata] = - useState(null); + const [error, setError] = useState(null); + const [startedReply, setStartedReply] = useState(false); + const [sources, setSources] = useState([]); const feedContainer: RefObject | undefined = useRef(null); + const { messages, input, handleInputChange, handleSubmit, isLoading } = + useChat({ + api: '/api/query-ai-handler', + onError: (error) => { + setError(error); + }, + onResponse: (_response) => { + setStartedReply(true); + sendCustomEvent('ai_query', 'ai', 'query', undefined, { + query: input, + }); + setError(null); + }, + onFinish: (response: Message) => { + setStartedReply(false); + setSources(extractLinksFromSourcesSection(response.content)); + // Here we have the message id and the timestamp, so we can create a linked list + }, + }); useEffect(() => { if (feedContainer.current) { @@ -39,64 +45,21 @@ export function FeedContainer(): JSX.Element { feedContainer.current.getElementsByClassName('feed-item'); elements[elements.length - 1].scrollIntoView({ behavior: 'smooth' }); } - }, [chatHistory, isLoading]); - - const handleSubmit = async (query: string, currentHistory: ChatItem[]) => { - if (!query) return; - - currentHistory.push({ role: 'user', content: query }); - - setIsLoading(true); - setQueryError(null); - - try { - const lastAnswerChatItem = - currentHistory.filter((item) => item.role === 'assistant').pop() || - null; - // Use previous assistant's answer if it exists - const aiResponse = await queryAi( - query, - lastAnswerChatItem ? lastAnswerChatItem.content : '' - ); - // TODO: Save a list of metadata corresponding to each query - // Saving Metadata for usage like feedback and analytics - setLastQueryMetadata({ - sources: aiResponse.sources - ? aiResponse.sources.map((source) => source.url) - : [], - textResponse: aiResponse.textResponse, - usage: aiResponse.usage || null, - }); - let content = aiResponse.textResponse; - if (aiResponse.sourcesMarkdown.length !== 0) - content += formatMarkdownSources(aiResponse.sourcesMarkdown); - - // Saving the new chat history used by AI for follow-up prompts - setChatHistory([ - ...getProcessedHistory(), - { role: 'assistant', content }, - ]); - - sendCustomEvent('ai_query', 'ai', 'query', undefined, { - query, - ...aiResponse.usage, - }); - } catch (error: any) { - setQueryError(error); - } - - setIsLoading(false); - }; + }, [messages, isLoading]); const handleFeedback = (statement: 'good' | 'bad', chatItemIndex: number) => { - const question = chatHistory[chatItemIndex - 1]; - const answer = chatHistory[chatItemIndex]; + // TODO(katerina): Fix this - Read on + // This is wrong + // We have to make sure to send the query for the actual message that was clicked + // Here we are just sending the last one + const question = messages[chatItemIndex - 1]; + const answer = messages[chatItemIndex]; sendCustomEvent('ai_feedback', 'ai', statement, undefined, { query: question ? question.content : 'Could not retrieve the question', result: answer ? answer.content : 'Could not retrieve the answer', - sources: lastQueryMetadata - ? JSON.stringify(lastQueryMetadata.sources) + sources: sources + ? JSON.stringify(sources) : 'Could not retrieve last answer sources', }); }; @@ -122,20 +85,21 @@ export function FeedContainer(): JSX.Element { className="relative" > handleFeedback(statement, chatItemIndex) } /> - {isLoading && } - {queryError && } + {/* Change this message if it's loading but it's writing as well */} + {isLoading && !startedReply && } + {error && }
handleSubmit(query, chatHistory)} + handleSubmit={handleSubmit} + handleInputChange={handleInputChange} + input={input} isDisabled={isLoading} />
diff --git a/nx-dev/feature-ai/src/lib/feed/feed-answer.tsx b/nx-dev/feature-ai/src/lib/feed/feed-answer.tsx index 3107efe53439b6..678fb96ea834ec 100644 --- a/nx-dev/feature-ai/src/lib/feed/feed-answer.tsx +++ b/nx-dev/feature-ai/src/lib/feed/feed-answer.tsx @@ -2,12 +2,14 @@ import { HandThumbDownIcon, HandThumbUpIcon, } from '@heroicons/react/24/outline'; -import { renderMarkdown } from '@nx/nx-dev/ui-markdoc'; import { cx } from '@nx/nx-dev/ui-primitives'; -import Link from 'next/link'; import { useState } from 'react'; import { ChatGptLogo } from './chat-gpt-logo'; -import { NrwlLogo } from './nrwl-logo'; +import ReactMarkdown from 'react-markdown'; +import { renderMarkdown } from '@nx/nx-dev/ui-markdoc'; + +const callout: string = + '{% callout type="warning" title="Always double-check!" %}The results may not be accurate, so please always double check with our documentation.{% /callout %}\n'; export function FeedAnswer({ content, @@ -60,7 +62,8 @@ export function FeedAnswer({

- {renderMarkdown(content, { filePath: '' }).node} + {!isFirst && renderMarkdown(callout, { filePath: '' }).node} +
{!isFirst && (
diff --git a/nx-dev/feature-ai/src/lib/feed/feed.tsx b/nx-dev/feature-ai/src/lib/feed/feed.tsx index 2e1b6cb74b5b68..94699b02897985 100644 --- a/nx-dev/feature-ai/src/lib/feed/feed.tsx +++ b/nx-dev/feature-ai/src/lib/feed/feed.tsx @@ -1,4 +1,4 @@ -import { ChatItem } from '@nx/nx-dev/data-access-ai'; +import { ChatItem } from '@nx/nx-dev/util-ai'; import { FeedAnswer } from './feed-answer'; import { FeedQuestion } from './feed-question'; diff --git a/nx-dev/feature-ai/src/lib/prompt.tsx b/nx-dev/feature-ai/src/lib/prompt.tsx index 762931a45b21ec..64fb78e4c69e10 100644 --- a/nx-dev/feature-ai/src/lib/prompt.tsx +++ b/nx-dev/feature-ai/src/lib/prompt.tsx @@ -1,18 +1,27 @@ -import { useEffect, useRef, useState } from 'react'; +import { ChangeEvent, FormEvent, useEffect, useRef } from 'react'; import { PaperAirplaneIcon } from '@heroicons/react/24/outline'; import { Button } from '@nx/nx-dev/ui-common'; import Textarea from 'react-textarea-autosize'; +import { ChatRequestOptions } from 'ai'; export function Prompt({ isDisabled, handleSubmit, + handleInputChange, + input, }: { isDisabled: boolean; - handleSubmit: (query: string) => void; + handleSubmit: ( + e: FormEvent, + chatRequestOptions?: ChatRequestOptions | undefined + ) => void; + handleInputChange: ( + e: ChangeEvent | ChangeEvent + ) => void; + input: string; }) { const formRef = useRef(null); const inputRef = useRef(null); - const [inputValue, setInputValue] = useState(''); useEffect(() => { if (inputRef.current) { @@ -23,13 +32,7 @@ export function Prompt({ return (
{ - event.preventDefault(); - if (!inputValue?.trim()) return; - handleSubmit(inputValue); - setInputValue(''); - event.currentTarget.reset(); - }} + onSubmit={handleSubmit} className="relative flex gap-2 max-w-2xl mx-auto py-0 px-2 shadow-lg rounded-md border border-slate-300 bg-white dark:border-slate-900 dark:bg-slate-700" >
@@ -45,7 +48,8 @@ export function Prompt({ } }} ref={inputRef} - onChange={(event) => setInputValue(event.target.value)} + value={input} + onChange={handleInputChange} id="query-prompt" name="query" disabled={isDisabled} diff --git a/nx-dev/feature-ai/src/lib/utils.ts b/nx-dev/feature-ai/src/lib/utils.ts deleted file mode 100644 index 9dc7bb44aece9b..00000000000000 --- a/nx-dev/feature-ai/src/lib/utils.ts +++ /dev/null @@ -1,7 +0,0 @@ -export function formatMarkdownSources(sourcesMarkdown: string): string { - return `\n -{% callout type="info" title="Sources" %} -${sourcesMarkdown} -{% /callout %} -\n`; -} diff --git a/nx-dev/nx-dev/pages/api/query-ai-handler.ts b/nx-dev/nx-dev/pages/api/query-ai-handler.ts index de801d5cde7beb..28a400d6a597db 100644 --- a/nx-dev/nx-dev/pages/api/query-ai-handler.ts +++ b/nx-dev/nx-dev/pages/api/query-ai-handler.ts @@ -1,26 +1,24 @@ -// based on: -// https://github.com/supabase-community/nextjs-openai-doc-search/blob/main/pages/api/vector-search.ts - import { NextRequest } from 'next/server'; import { + ChatItem, CustomError, DEFAULT_MATCH_COUNT, DEFAULT_MATCH_THRESHOLD, MIN_CONTENT_LENGTH, PROMPT, PageSection, + appendToStream, checkEnvVariables, - getListOfSources, - getMessageFromResponse, + formatMarkdownSources, initializeChat, - moderateContent, - openAiAPICall, - sanitizeLinksInResponse, - toMarkdownList, + removeSourcesSection, + // moderateContent, } from '@nx/nx-dev/util-ai'; import { SupabaseClient, createClient } from '@supabase/supabase-js'; -import { CreateCompletionResponseUsage, CreateEmbeddingResponse } from 'openai'; +import OpenAI from 'openai'; +import { OpenAIStream, StreamingTextResponse } from 'ai'; import GPT3Tokenizer from 'gpt3-tokenizer'; +import { Stream } from 'openai/streaming'; const supabaseUrl = process.env['NX_NEXT_PUBLIC_SUPABASE_URL']; const supabaseServiceKey = process.env['NX_SUPABASE_SERVICE_ROLE_KEY_ACTUAL']; @@ -33,65 +31,57 @@ export const config = { export default async function handler(request: NextRequest) { try { checkEnvVariables(openAiKey, supabaseUrl, supabaseServiceKey); - const { query, aiResponse, chatFullHistory } = await request.json(); + const openai = new OpenAI({ + apiKey: openAiKey, // This is also the default, can be omitted + }); + + const { messages } = (await request.json()) as { messages: ChatItem[] }; const supabaseClient: SupabaseClient = createClient( supabaseUrl as string, supabaseServiceKey as string ); + let query: string | null = null; + if (messages?.length > 0) { + const lastMessage = messages[messages.length - 1]; + if (lastMessage?.role === 'user') { + query = lastMessage.content; + } + } + if (!query) { throw new CustomError('user_error', 'Missing query in request data', { missing_query: true, }); } - - // Moderate the content to comply with OpenAI T&C const sanitizedQuery = query.trim(); - await moderateContent(sanitizedQuery, openAiKey as string); - - // Create embedding from query - // NOTE: Here, we may or may not want to include the previous AI response - /** - * For retrieving relevant Nx documentation sections via embeddings, it's a design decision. - * Including the prior response might give more contextually relevant sections, - * but just sending the query might suffice for many cases. - * - * We can experiment with this. - * - * How the solution looks like with previous response: - * - * const embeddingResponse = await openAiCall( - * { input: sanitizedQuery + aiResponse }, - * 'embedding' - * ); - * - * This costs more tokens, so if we see costs skyrocket we remove it. - * As it says in the docs, it's a design decision, and it may or may not really improve results. - */ - const embeddingResponseObj = await openAiAPICall( - { input: sanitizedQuery + aiResponse, model: 'text-embedding-ada-002' }, - 'embedding', - openAiKey as string - ); - - const embeddingResponse = await embeddingResponseObj.json(); - - if (!embeddingResponseObj.ok) { - throw new CustomError( - 'application_error', - 'Failed to create embedding for question', - { - data: embeddingResponse, - } - ); - } + // Moderate the content to comply with OpenAI T&C + // Removing the moderation for now + // to see if it's faster + // await moderateContent(sanitizedQuery, openai); + + // We include the previous response, + // to make sure the embeddings (doc sections) + // we get back are relevant. + const lastAiResponse = messages + .filter((message) => message.role === 'assistant') + .map((message) => message.content) + .pop(); + + const embeddingResponse: OpenAI.Embeddings.CreateEmbeddingResponse = + await openai.embeddings.create({ + model: 'text-embedding-ada-002', + input: sanitizedQuery + removeSourcesSection(lastAiResponse ?? ''), + }); const { data: [{ embedding }], - }: CreateEmbeddingResponse = embeddingResponse; + } = embeddingResponse; + // Based on: + // https://github.com/supabase-community/nextjs-openai-doc-search/blob/main/pages/api/vector-search.ts const { error: matchError, data: pageSections } = await supabaseClient.rpc( 'match_page_sections_2', { @@ -135,63 +125,28 @@ export default async function handler(request: NextRequest) { contextText += `${content.trim()}\n---\n`; } - const { chatMessages: chatGptMessages, chatHistory } = initializeChat( - chatFullHistory, + const { chatMessages } = initializeChat( + messages, query, contextText, - PROMPT, - aiResponse + PROMPT ); - const responseObj = await openAiAPICall( - { + const response: Stream = + await openai.chat.completions.create({ model: 'gpt-3.5-turbo-16k', - messages: chatGptMessages, + messages: chatMessages, temperature: 0, - stream: false, - }, - 'chatCompletion', - openAiKey as string - ); + stream: true, + }); - const response = await responseObj.json(); + const sourcesMarkdown = formatMarkdownSources(pageSections); + const stream = OpenAIStream(response); + const finalStream = await appendToStream(stream, sourcesMarkdown); - if (!responseObj.ok) { - throw new CustomError( - 'application_error', - 'Failed to generate completion', - { - data: response, - } - ); - } - // Message asking to double-check - const callout: string = - '{% callout type="warning" title="Always double-check!" %}The results may not be accurate, so please always double check with our documentation.{% /callout %}\n'; - // Append the warning message asking to double-check! - const message = [callout, getMessageFromResponse(response)].join(''); - - const responseWithoutBadLinks = await sanitizeLinksInResponse(message); - - const sources = getListOfSources(pageSections); - - const responseData = { - textResponse: responseWithoutBadLinks, - usage: response.usage as CreateCompletionResponseUsage, - sources, - sourcesMarkdown: toMarkdownList(sources), - chatHistory, - requestTokens: response.usage?.total_tokens, - }; - return new Response(JSON.stringify(responseData), { - status: 200, - headers: { - 'content-type': 'application/json', - }, - }); + return new StreamingTextResponse(finalStream); } catch (err: unknown) { console.error('Error: ', err); - return new Response( JSON.stringify({ ...JSON.parse(JSON.stringify(err)), diff --git a/nx-dev/util-ai/package.json b/nx-dev/util-ai/package.json index aa059580b8e195..74410b250a84eb 100644 --- a/nx-dev/util-ai/package.json +++ b/nx-dev/util-ai/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "dependencies": { "tslib": "^2.3.0", - "openai": "~3.3.0" + "openai": "~4.3.1" }, "type": "commonjs", "main": "./src/index.js", diff --git a/nx-dev/util-ai/src/index.ts b/nx-dev/util-ai/src/index.ts index 2941e8aa809a65..6edc5fa33a64cd 100644 --- a/nx-dev/util-ai/src/index.ts +++ b/nx-dev/util-ai/src/index.ts @@ -1,5 +1,4 @@ export * from './lib/utils'; export * from './lib/constants'; -export * from './lib/openai-call'; export * from './lib/moderation'; export * from './lib/chat-utils'; diff --git a/nx-dev/util-ai/src/lib/chat-utils.ts b/nx-dev/util-ai/src/lib/chat-utils.ts index 522139c4d2844e..026438b0f70092 100644 --- a/nx-dev/util-ai/src/lib/chat-utils.ts +++ b/nx-dev/util-ai/src/lib/chat-utils.ts @@ -1,28 +1,21 @@ -import { - ChatCompletionRequestMessageRoleEnum, - CreateChatCompletionResponse, -} from 'openai'; +import OpenAI from 'openai'; import { ChatItem, PageSection } from './utils'; /** * Initializes a chat session by generating the initial chat messages based on the given parameters. * - * @param {ChatItem[]} chatFullHistory - The full chat history. + * @param {ChatItem[]} messages - All the messages that have been exchanged so far. * @param {string} query - The user's query. * @param {string} contextText - The context text or Nx Documentation. * @param {string} prompt - The prompt message displayed to the user. - * @param {string} [aiResponse] - The AI assistant's response. - * @returns {Object} - An object containing the generated chat messages and updated chat history. - * - chatMessages: An array of chat messages for the chat session. - * - chatHistory: The updated chat history. + * @returns {Object} - An object containing the generated chat messages */ export function initializeChat( - chatFullHistory: ChatItem[], + messages: ChatItem[], query: string, contextText: string, - prompt: string, - aiResponse?: string -): { chatMessages: ChatItem[]; chatHistory: ChatItem[] } { + prompt: string +): { chatMessages: ChatItem[] } { const finalQuery = ` You will be provided the Nx Documentation. Answer my message provided by following the approach below: @@ -31,81 +24,34 @@ export function initializeChat( - Step 2: Deduce the diagnostic REASONING process from the premises (clues, question), relying ONLY on the information provided in the Nx Documentation. If you recognize vulgar language, answer the question if possible, and educate the user to stay polite. - Step 3: EVALUATE the reasoning. If the reasoning aligns with the Nx Documentation, accept it. Do not use any external knowledge or make assumptions outside of the provided Nx documentation. If the reasoning doesn't strictly align with the Nx Documentation or relies on external knowledge or inference, reject it and answer with the exact string: "Sorry, I don't know how to help with that. You can visit the [Nx documentation](https://nx.dev/getting-started/intro) for more info." - - Final Step: You can also rely on the messages we have exchanged so far. Do NOT reveal the approach to the user. + - Final Step: You can also rely on the messages we have exchanged so far. Do NOT reveal this approach or the steps to the user. Only provide the answer. Start replying with the answer directly. + Nx Documentation: ${contextText} ---- My message: ${query} `; - let chatGptMessages: ChatItem[] = []; - let messages: ChatItem[] = []; - - if (chatFullHistory.length > 0) { - messages = [ - { - role: ChatCompletionRequestMessageRoleEnum.Assistant, - content: aiResponse ?? '', - }, - { role: ChatCompletionRequestMessageRoleEnum.User, content: finalQuery }, - ]; - chatGptMessages = [...chatFullHistory, ...messages]; - } else { - messages = [ - { role: ChatCompletionRequestMessageRoleEnum.System, content: prompt }, - { role: ChatCompletionRequestMessageRoleEnum.User, content: finalQuery }, - ]; - chatGptMessages = [...messages]; - } - chatFullHistory.push(...messages); + // Remove the last message, which is the user query + // and restructure the user query to include the instructions and context. + // Add the system prompt as the first message of the array + // and add the user query as the last message of the array. + messages.pop(); + messages = [ + { role: 'system', content: prompt }, + ...(messages ?? []), + { role: 'user', content: finalQuery }, + ]; - return { chatMessages: chatGptMessages, chatHistory: chatFullHistory }; + return { chatMessages: messages }; } export function getMessageFromResponse( - response: CreateChatCompletionResponse + response: OpenAI.Chat.Completions.ChatCompletion ): string { return response.choices[0].message?.content ?? ''; } -export async function sanitizeLinksInResponse( - response: string -): Promise { - const regex = /https:\/\/nx\.dev[^) \n]*[^).]/g; - const urls = response.match(regex); - - if (urls) { - for (const url of urls) { - const linkIsWrong = await is404(url); - if (linkIsWrong) { - response = response.replace( - url, - 'https://nx.dev/getting-started/intro' - ); - } - } - } - - return response; -} - -async function is404(url: string): Promise { - try { - const response = await fetch(url.replace('https://nx.dev', '')); - if (response.status === 404) { - return true; - } else { - return false; - } - } catch (error) { - if ((error as any)?.response?.status === 404) { - return true; - } else { - return false; - } - } -} - export function getListOfSources( pageSections: PageSection[] ): { heading: string; url: string }[] { @@ -133,6 +79,17 @@ export function getListOfSources( return result; } +export function formatMarkdownSources(pageSections: PageSection[]): string { + const sources = getListOfSources(pageSections); + const sourcesMarkdown = toMarkdownList(sources); + + return `\n +### Sources + +${sourcesMarkdown} +\n`; +} + export function toMarkdownList( sections: { heading: string; url: string }[] ): string { @@ -140,3 +97,48 @@ export function toMarkdownList( .map((section) => `- [${section.heading}](${section.url})`) .join('\n'); } + +export function extractLinksFromSourcesSection(markdown: string): string[] { + const sectionRegex = /### Sources\n\n([\s\S]*?)(?:\n##|$)/; + const sectionMatch = sectionRegex.exec(markdown); + + if (!sectionMatch) return []; + + const sourcesSection = sectionMatch[1]; + + const linkRegex = /\]\((.*?)\)/g; + const links: string[] = []; + let match; + + while ((match = linkRegex.exec(sourcesSection)) !== null) { + links.push(match[1]); + } + + return links; +} + +export function removeSourcesSection(markdown: string): string { + const sectionRegex = /### Sources\n\n([\s\S]*?)(?:\n###|$)/; + return markdown.replace(sectionRegex, '').trim(); +} + +export async function appendToStream( + originalStream: ReadableStream, + appendContent: string +): Promise> { + let controller: ReadableStreamDefaultController; + return new ReadableStream({ + async start(ctrl) { + controller = ctrl; + const reader = originalStream.getReader(); + + let result; + while (!(result = await reader.read()).done) { + controller.enqueue(result.value); + } + + controller.enqueue(new TextEncoder().encode(appendContent)); + controller.close(); + }, + }); +} diff --git a/nx-dev/util-ai/src/lib/moderation.ts b/nx-dev/util-ai/src/lib/moderation.ts index 73240ddb607209..893f919cc014ff 100644 --- a/nx-dev/util-ai/src/lib/moderation.ts +++ b/nx-dev/util-ai/src/lib/moderation.ts @@ -1,23 +1,21 @@ -import { openAiAPICall } from './openai-call'; +import OpenAI from 'openai'; import { CustomError } from './utils'; -export async function moderateContent( - sanitizedQuery: string, - openAiKey: string -) { - const moderationResponseObj = await openAiAPICall( - { input: sanitizedQuery }, - 'moderation', - openAiKey as string - ); +export async function moderateContent(sanitizedQuery: string, openai: OpenAI) { + try { + const moderationResponse = await openai.moderations.create({ + input: sanitizedQuery, + }); - const moderationResponse = await moderationResponseObj.json(); - const [results] = moderationResponse.results; + const [results] = moderationResponse.results; - if (results.flagged) { - throw new CustomError('user_error', 'Flagged content', { - flagged: true, - categories: results.categories, - }); + if (results.flagged) { + throw new CustomError('user_error', 'Flagged content', { + flagged: true, + categories: results.categories, + }); + } + } catch (e) { + console.error('Error when moderating content: ', e); } } diff --git a/nx-dev/util-ai/src/lib/openai-call.ts b/nx-dev/util-ai/src/lib/openai-call.ts deleted file mode 100644 index 9916848ce52483..00000000000000 --- a/nx-dev/util-ai/src/lib/openai-call.ts +++ /dev/null @@ -1,26 +0,0 @@ -export async function openAiAPICall( - input: object, - action: 'moderation' | 'embedding' | 'chatCompletion', - openAiKey: string -) { - let apiUrl = 'https://api.openai.com/v1/'; - - if (action === 'embedding') { - apiUrl += 'embeddings'; - } else if (action === 'chatCompletion') { - apiUrl += 'chat/completions'; - } else if (action === 'moderation') { - apiUrl += 'moderations'; - } else { - return new Response('Invalid action', { status: 400 }); - } - - return fetch(apiUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${openAiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(input), - }); -} diff --git a/nx-dev/util-ai/src/lib/utils.ts b/nx-dev/util-ai/src/lib/utils.ts index 6a8a6b5ba58363..889d3faf575110 100644 --- a/nx-dev/util-ai/src/lib/utils.ts +++ b/nx-dev/util-ai/src/lib/utils.ts @@ -1,5 +1,3 @@ -import { ChatCompletionRequestMessageRoleEnum } from 'openai'; - export function checkEnvVariables( openAiKey?: string, supabaseUrl?: string, @@ -57,6 +55,6 @@ export interface PageSection { } export interface ChatItem { - role: ChatCompletionRequestMessageRoleEnum; + role: 'system' | 'user' | 'assistant' | 'function'; content: string; } diff --git a/package.json b/package.json index 7046a3cbf135a3..4315ba3e3ef733 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,10 @@ "@ngrx/router-store": "~16.0.0", "@ngrx/store": "~16.0.0", "@nguniversal/builders": "~16.2.0", + "@nx/angular": "16.8.0-beta.3", "@nx/cypress": "16.8.0-beta.3", "@nx/devkit": "16.8.0-beta.3", + "@nx/esbuild": "16.8.0-beta.3", "@nx/eslint-plugin": "16.8.0-beta.3", "@nx/jest": "16.8.0-beta.3", "@nx/js": "16.8.0-beta.3", @@ -77,8 +79,6 @@ "@nx/storybook": "16.8.0-beta.3", "@nx/web": "16.8.0-beta.3", "@nx/webpack": "16.8.0-beta.3", - "@nx/esbuild": "16.8.0-beta.3", - "@nx/angular": "16.8.0-beta.3", "@parcel/watcher": "2.0.4", "@phenomnomnominal/tsquery": "~5.0.1", "@playwright/test": "^1.36.1", @@ -134,7 +134,7 @@ "@xstate/immer": "0.3.1", "@xstate/inspect": "0.7.0", "@xstate/react": "3.0.1", - "ai": "^2.1.15", + "ai": "^2.2.10", "ajv": "^8.11.0", "autoprefixer": "10.4.13", "babel-jest": "29.4.3", @@ -230,7 +230,7 @@ "nx-cloud": "16.4.0-beta.6", "octokit": "^2.0.14", "open": "^8.4.0", - "openai": "~3.3.0", + "openai": "~4.3.1", "ora": "5.3.0", "parse-markdown-links": "^1.0.4", "parse5": "4.0.0", @@ -242,6 +242,7 @@ "prettier-plugin-tailwindcss": "^0.1.5", "pretty-quick": "^3.1.0", "raw-loader": "^4.0.2", + "react-markdown": "^8.0.7", "react-redux": "8.0.5", "react-refresh": "^0.10.0", "react-router-dom": "^6.11.2", @@ -374,4 +375,3 @@ } } } - diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8dba879b69242b..9f8852550f4d34 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -478,8 +478,8 @@ devDependencies: specifier: 3.0.1 version: 3.0.1(@types/react@18.2.14)(react@18.2.0)(xstate@4.34.0) ai: - specifier: ^2.1.15 - version: 2.1.15(react@18.2.0)(svelte@3.59.2)(vue@3.3.4) + specifier: ^2.2.10 + version: 2.2.10(react@18.2.0)(solid-js@1.7.11)(svelte@4.2.0)(vue@3.3.4) ajv: specifier: ^8.11.0 version: 8.11.0 @@ -766,8 +766,8 @@ devDependencies: specifier: ^8.4.0 version: 8.4.0 openai: - specifier: ~3.3.0 - version: 3.3.0 + specifier: ~4.3.1 + version: 4.3.1 ora: specifier: 5.3.0 version: 5.3.0 @@ -801,6 +801,9 @@ devDependencies: raw-loader: specifier: ^4.0.2 version: 4.0.2(webpack@5.88.0) + react-markdown: + specifier: ^8.0.7 + version: 8.0.7(@types/react@18.2.14)(react@18.2.0) react-redux: specifier: 8.0.5 version: 8.0.5(@types/react-dom@18.2.6)(@types/react@18.2.14)(react-dom@18.2.0)(react@18.2.0)(redux@4.2.0) @@ -9976,7 +9979,6 @@ packages: resolution: {integrity: sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==} dependencies: '@types/unist': 3.0.0 - dev: false /@types/hoist-non-react-statics@3.3.1: resolution: {integrity: sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA==} @@ -11287,16 +11289,19 @@ packages: indent-string: 4.0.0 dev: true - /ai@2.1.15(react@18.2.0)(svelte@3.59.2)(vue@3.3.4): - resolution: {integrity: sha512-ePxoo9yEpHrC6n2O5b0Ko9C0dZEEXBY9FuhbrR1PVgdo4cSislTqg9TSPdVKT3mnw01A2pEg24cQ8ikRyH9m4Q==} + /ai@2.2.10(react@18.2.0)(solid-js@1.7.11)(svelte@4.2.0)(vue@3.3.4): + resolution: {integrity: sha512-3FARCB9X57YxpAJeAUvZHTeeQ549B/kTMQk5Qet1rZNm9EjKXeHUiQfaq+L8v9f75HYasZXJIl//owzdjojTTw==} engines: {node: '>=14.6'} peerDependencies: react: ^18.2.0 - svelte: ^4.0.0 + solid-js: ^1.7.7 + svelte: ^3.0.0 || ^4.0.0 vue: ^3.3.4 peerDependenciesMeta: react: optional: true + solid-js: + optional: true svelte: optional: true vue: @@ -11304,12 +11309,19 @@ packages: dependencies: eventsource-parser: 1.0.0 nanoid: 3.3.6 + openai: 4.2.0 react: 18.2.0 - sswr: 1.10.0(svelte@3.59.2) - svelte: 3.59.2 - swr: 2.1.5(react@18.2.0) - swrv: 1.0.3(vue@3.3.4) + solid-js: 1.7.11 + solid-swr-store: 0.10.7(solid-js@1.7.11)(swr-store@0.10.6) + sswr: 2.0.0(svelte@4.2.0) + svelte: 4.2.0 + swr: 2.2.0(react@18.2.0) + swr-store: 0.10.6 + swrv: 1.0.4(vue@3.3.4) vue: 3.3.4 + transitivePeerDependencies: + - encoding + - supports-color dev: true /ajv-formats@2.1.1(ajv@8.11.0): @@ -11556,6 +11568,12 @@ packages: deep-equal: 2.0.5 dev: true + /aria-query@5.3.0: + resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==} + dependencies: + dequal: 2.0.3 + dev: true + /array-differ@3.0.0: resolution: {integrity: sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==} engines: {node: '>=8'} @@ -11784,14 +11802,6 @@ packages: - debug dev: true - /axios@0.26.1: - resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==} - dependencies: - follow-redirects: 1.15.2(debug@4.3.2) - transitivePeerDependencies: - - debug - dev: true - /axios@1.0.0: resolution: {integrity: sha512-SsHsGFN1qNPFT5QhSoSD37SHDfGyLSW5AESmyLk2JeCMHv5g0I9g0Hz/zQHx2KNe0jGXh2q2hAm7OdkXm360CA==} dependencies: @@ -11821,6 +11831,12 @@ packages: deep-equal: 2.0.5 dev: true + /axobject-query@3.2.1: + resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} + dependencies: + dequal: 2.0.3 + dev: true + /babel-jest@29.4.3(@babel/core@7.22.9): resolution: {integrity: sha512-o45Wyn32svZE+LnMVWv/Z4x0SwtLbh4FyGcYtR20kIWd+rdrDZ9Fzq8Ml3MYLD+mZvEdzCjZsCnYZ2jpJyQ+Nw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -12142,9 +12158,17 @@ packages: babel-preset-current-node-syntax: 1.0.1(@babel/core@7.22.9) dev: true + /bail@2.0.2: + resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} + dev: true + /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + /base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + dev: true + /base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} dev: true @@ -12847,6 +12871,10 @@ packages: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} dev: true + /charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + dev: true + /check-error@1.0.2: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} dev: false @@ -13031,6 +13059,16 @@ packages: engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} dev: true + /code-red@1.0.4: + resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + '@types/estree': 1.0.1 + acorn: 8.10.0 + estree-walker: 3.0.3 + periscopic: 3.1.0 + dev: true + /coffeescript@1.12.7: resolution: {integrity: sha512-pLXHFxQMPklVoEekowk8b3erNynC+DVJzChxS/LCBBgR6/8AJkHivkm//zbowcfc7BTCAjryuhx6gPqPRfsFoA==} engines: {node: '>=0.8.0'} @@ -13098,6 +13136,10 @@ packages: resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} dev: false + /comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + dev: true + /commander@11.0.0: resolution: {integrity: sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==} engines: {node: '>=16'} @@ -13663,6 +13705,10 @@ packages: which: 2.0.2 dev: true + /crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + dev: true + /crypto-random-string@2.0.0: resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==} engines: {node: '>=8'} @@ -14510,6 +14556,13 @@ packages: engines: {node: '>=0.3.1'} dev: true + /digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + dev: true + /dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -15532,6 +15585,12 @@ packages: resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} dev: true + /estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + dependencies: + '@types/estree': 1.0.1 + dev: true + /esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} @@ -16195,6 +16254,10 @@ packages: webpack: 5.88.0(@swc/core@1.3.51)(esbuild@0.19.2) dev: true + /form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + dev: true + /form-data@2.3.3: resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} engines: {node: '>= 0.12'} @@ -16226,6 +16289,14 @@ packages: engines: {node: '>=0.4.x'} dev: false + /formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + dev: true + /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -16913,6 +16984,10 @@ packages: resolution: {integrity: sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==} dev: false + /hast-util-whitespace@2.0.1: + resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} + dev: true + /hastscript@6.0.0: resolution: {integrity: sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==} dependencies: @@ -17471,6 +17546,10 @@ packages: tslib: 2.5.0 dev: true + /inline-style-parser@0.1.1: + resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} + dev: true + /inquirer@7.3.3: resolution: {integrity: sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==} engines: {node: '>=8.0.0'} @@ -17619,6 +17698,15 @@ packages: has-tostringtag: 1.0.0 dev: true + /is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + dev: true + + /is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + dev: true + /is-builtin-module@3.2.0: resolution: {integrity: sha512-phDA4oSGt7vl1n5tJvTWooWWAsXLY+2xCnxNqvKhGEzujg+A43wPlPOyDg3C8XQHN+6k/JTQWJ/j0dQh/qr+Hw==} engines: {node: '>=6'} @@ -17782,6 +17870,11 @@ packages: engines: {node: '>=10'} dev: true + /is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + dev: true + /is-plain-object@2.0.4: resolution: {integrity: sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==} engines: {node: '>=0.10.0'} @@ -17812,6 +17905,12 @@ packages: '@types/estree': 1.0.1 dev: true + /is-reference@3.0.1: + resolution: {integrity: sha512-baJJdQLiYaJdvFbJqXrcGv3WU3QCzBlUcI5QhbesIm6/xPsvmO+2CDoi/GMOFBQEQm+PXkwOPrp9KK5ozZsp2w==} + dependencies: + '@types/estree': 1.0.1 + dev: true + /is-regex@1.1.4: resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} engines: {node: '>= 0.4'} @@ -19299,6 +19398,10 @@ packages: - supports-color dev: true + /locate-character@3.0.0: + resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} + dev: true + /locate-path@2.0.0: resolution: {integrity: sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==} engines: {node: '>=4'} @@ -19671,12 +19774,28 @@ packages: blueimp-md5: 2.19.0 dev: false + /md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + dev: true + /mdast-util-definitions@4.0.0: resolution: {integrity: sha512-k8AJ6aNnUkB7IE+5azR9h81O5EQ/cTDXtWdMq9Kk5KcEW/8ritU5CeLg/9HhOC++nALHBlaogJ5jz0Ybk3kPMQ==} dependencies: unist-util-visit: 2.0.3 dev: true + /mdast-util-definitions@5.1.2: + resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==} + dependencies: + '@types/mdast': 3.0.12 + '@types/unist': 2.0.6 + unist-util-visit: 4.1.2 + dev: true + /mdast-util-from-markdown@1.3.1: resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} dependencies: @@ -19703,6 +19822,19 @@ packages: unist-util-is: 5.2.1 dev: true + /mdast-util-to-hast@12.3.0: + resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} + dependencies: + '@types/hast': 2.3.4 + '@types/mdast': 3.0.12 + mdast-util-definitions: 5.1.2 + micromark-util-sanitize-uri: 1.2.0 + trim-lines: 3.0.1 + unist-util-generated: 2.0.1 + unist-util-position: 4.0.4 + unist-util-visit: 4.1.2 + dev: true + /mdast-util-to-markdown@1.5.0: resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==} dependencies: @@ -20846,6 +20978,11 @@ packages: minimatch: 3.0.5 dev: true + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: true + /node-emoji@1.11.0: resolution: {integrity: sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==} dependencies: @@ -21554,13 +21691,38 @@ packages: is-wsl: 2.2.0 dev: true - /openai@3.3.0: - resolution: {integrity: sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==} + /openai@4.2.0: + resolution: {integrity: sha512-zfvpO2eITIxIjTG8T6Cek7NB2dMvP/LW0TRUJ4P9E8+qbBNKw00DrtfF64b+fAV2+wUYCVyynT6iSycJ//TtbA==} + hasBin: true dependencies: - axios: 0.26.1 - form-data: 4.0.0 + '@types/node': 18.16.9 + '@types/node-fetch': 2.6.4 + abort-controller: 3.0.0 + agentkeepalive: 4.2.1 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.6.12 transitivePeerDependencies: - - debug + - encoding + - supports-color + dev: true + + /openai@4.3.1: + resolution: {integrity: sha512-64iI2LbJLk0Ss4Nv5IrdGFe6ALNnKlMuXoGuH525bJYxdupJfDCAtra/Jigex1z8it0U82M87tR2TMGU+HYeFQ==} + hasBin: true + dependencies: + '@types/node': 18.16.9 + '@types/node-fetch': 2.6.4 + abort-controller: 3.0.0 + agentkeepalive: 4.2.1 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.6.12 + transitivePeerDependencies: + - encoding + - supports-color dev: true /opener@1.5.2: @@ -22063,6 +22225,14 @@ packages: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} dev: true + /periscopic@3.1.0: + resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} + dependencies: + '@types/estree': 1.0.1 + estree-walker: 3.0.3 + is-reference: 3.0.1 + dev: true + /picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -23517,6 +23687,10 @@ packages: xtend: 4.0.2 dev: false + /property-information@6.2.0: + resolution: {integrity: sha512-kma4U7AFCTwpqq5twzC1YVIDXSqg6qQK6JN0smOw8fgRy1OkMi0CYSzFmsy6dnqSenamAtj0CyXMUJ1Mf6oROg==} + dev: true + /proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} dev: true @@ -23757,6 +23931,33 @@ packages: resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} dev: true + /react-markdown@8.0.7(@types/react@18.2.14)(react@18.2.0): + resolution: {integrity: sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + dependencies: + '@types/hast': 2.3.4 + '@types/prop-types': 15.7.5 + '@types/react': 18.2.14 + '@types/unist': 2.0.6 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 2.0.1 + prop-types: 15.8.1 + property-information: 6.2.0 + react: 18.2.0 + react-is: 18.2.0 + remark-parse: 10.0.2 + remark-rehype: 10.1.0 + space-separated-tokens: 2.0.2 + style-to-object: 0.4.2 + unified: 10.1.2 + unist-util-visit: 4.1.2 + vfile: 5.3.7 + transitivePeerDependencies: + - supports-color + dev: true + /react-redux@8.0.5(@types/react-dom@18.2.6)(@types/react@18.2.14)(react-dom@18.2.0)(react@18.2.0)(redux@4.2.0): resolution: {integrity: sha512-Q2f6fCKxPFpkXt1qNRZdEDLlScsDWyrgSj0mliK59qU6W5gvBiKkdMEG2lJzhd1rCctf0hb6EtePPLZ2e0m1uw==} peerDependencies: @@ -24217,6 +24418,25 @@ packages: unist-util-visit: 2.0.3 dev: true + /remark-parse@10.0.2: + resolution: {integrity: sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw==} + dependencies: + '@types/mdast': 3.0.12 + mdast-util-from-markdown: 1.3.1 + unified: 10.1.2 + transitivePeerDependencies: + - supports-color + dev: true + + /remark-rehype@10.1.0: + resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} + dependencies: + '@types/hast': 2.3.4 + '@types/mdast': 3.0.12 + mdast-util-to-hast: 12.3.0 + unified: 10.1.2 + dev: true + /remark-slug@6.1.0: resolution: {integrity: sha512-oGCxDF9deA8phWvxFuyr3oSJsdyUAxMFbA0mZ7Y1Sas+emILtO+e5WutF9564gDsEN4IXaQXm5pFo6MLH+YmwQ==} dependencies: @@ -24893,6 +25113,11 @@ packages: randombytes: 2.1.0 dev: true + /seroval@0.5.1: + resolution: {integrity: sha512-ZfhQVB59hmIauJG5Ydynupy8KHyr5imGNtdDhbZG68Ufh1Ynkv9KOYOAABf71oVbQxJ8VkWnMHAjEHE7fWkH5g==} + engines: {node: '>=10'} + dev: true + /serve-favicon@2.5.0: resolution: {integrity: sha512-FMW2RvqNr03x+C0WxTyu6sOv21oOjkq5j8tjquWccwa6ScNyGFOGJVpuS1NmTVGBAHS07xnSKotgf2ehQmf9iA==} engines: {node: '>= 0.8.0'} @@ -25188,6 +25413,24 @@ packages: smart-buffer: 4.2.0 dev: true + /solid-js@1.7.11: + resolution: {integrity: sha512-JkuvsHt8jqy7USsy9xJtT18aF9r2pFO+GB8JQ2XGTvtF49rGTObB46iebD25sE3qVNvIbwglXOXdALnJq9IHtQ==} + dependencies: + csstype: 3.1.1 + seroval: 0.5.1 + dev: true + + /solid-swr-store@0.10.7(solid-js@1.7.11)(swr-store@0.10.6): + resolution: {integrity: sha512-A6d68aJmRP471aWqKKPE2tpgOiR5fH4qXQNfKIec+Vap+MGQm3tvXlT8n0I8UgJSlNAsSAUuw2VTviH2h3Vv5g==} + engines: {node: '>=10'} + peerDependencies: + solid-js: ^1.2 + swr-store: ^0.10 + dependencies: + solid-js: 1.7.11 + swr-store: 0.10.6 + dev: true + /sonic-boom@1.4.1: resolution: {integrity: sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==} dependencies: @@ -25291,6 +25534,10 @@ packages: /space-separated-tokens@1.1.5: resolution: {integrity: sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==} + /space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + dev: true + /spdx-compare@1.0.0: resolution: {integrity: sha512-C1mDZOX0hnu0ep9dfmuoi03+eOdDoz2yvK79RxbcrVEG1NO1Ph35yW102DHWKN4pk80nwCgeMmSY5L25VE4D9A==} dependencies: @@ -25400,13 +25647,13 @@ packages: minipass: 3.3.4 dev: true - /sswr@1.10.0(svelte@3.59.2): - resolution: {integrity: sha512-nLWAJSQy3h8t7rrbTXanRyVHuQPj4PwKIVGe4IMlxJFdhyaxnN/JGACnvQKGDeWiTGYIZIx/jRuUsPEF0867Pg==} + /sswr@2.0.0(svelte@4.2.0): + resolution: {integrity: sha512-mV0kkeBHcjcb0M5NqKtKVg/uTIYNlIIniyDfSGrSfxpEdM9C365jK0z55pl9K0xAkNTJi2OAOVFQpgMPUk+V0w==} peerDependencies: - svelte: ^3.29.0 + svelte: ^4.0.0 dependencies: - svelte: 3.59.2 - swrev: 3.0.0 + svelte: 4.2.0 + swrev: 4.0.0 dev: true /stable@0.1.8: @@ -25684,6 +25931,12 @@ packages: webpack: 5.88.0(@swc/core@1.3.51)(esbuild@0.19.2) dev: true + /style-to-object@0.4.2: + resolution: {integrity: sha512-1JGpfPB3lo42ZX8cuPrheZbfQ6kqPPnPHlKMyeRYtfKD+0jG+QsXgXN57O/dvJlzlB2elI6dGmrPnl5VPQFPaA==} + dependencies: + inline-style-parser: 0.1.1 + dev: true + /style-value-types@4.1.4: resolution: {integrity: sha512-LCJL6tB+vPSUoxgUBt9juXIlNJHtBMy8jkXzUJSBzeHWdBu6lhzHqCvLVkXFGsFIlNa2ln1sQHya/gzaFmB2Lg==} dependencies: @@ -25808,9 +26061,23 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} - /svelte@3.59.2: - resolution: {integrity: sha512-vzSyuGr3eEoAtT/A6bmajosJZIUWySzY2CzB3w2pgPvnkUjGqlDnsNnA0PMO+mMAhuyMul6C2uuZzY6ELSkzyA==} - engines: {node: '>= 8'} + /svelte@4.2.0: + resolution: {integrity: sha512-kVsdPjDbLrv74SmLSUzAsBGquMs4MPgWGkGLpH+PjOYnFOziAvENVzgJmyOCV2gntxE32aNm8/sqNKD6LbIpeQ==} + engines: {node: '>=16'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.18 + acorn: 8.10.0 + aria-query: 5.3.0 + axobject-query: 3.2.1 + code-red: 1.0.4 + css-tree: 2.3.1 + estree-walker: 3.0.3 + is-reference: 3.0.1 + locate-character: 3.0.0 + magic-string: 0.30.2 + periscopic: 3.1.0 dev: true /svg-parser@2.0.4: @@ -25858,8 +26125,15 @@ packages: webpack: 5.88.0(@swc/core@1.3.51)(esbuild@0.19.2) dev: true - /swr@2.1.5(react@18.2.0): - resolution: {integrity: sha512-/OhfZMcEpuz77KavXST5q6XE9nrOBOVcBLWjMT+oAE/kQHyE3PASrevXCtQDZ8aamntOfFkbVJp7Il9tNBQWrw==} + /swr-store@0.10.6: + resolution: {integrity: sha512-xPjB1hARSiRaNNlUQvWSVrG5SirCjk2TmaUyzzvk69SZQan9hCJqw/5rG9iL7xElHU784GxRPISClq4488/XVw==} + engines: {node: '>=10'} + dependencies: + dequal: 2.0.3 + dev: true + + /swr@2.2.0(react@18.2.0): + resolution: {integrity: sha512-AjqHOv2lAhkuUdIiBu9xbuettzAzWXmCEcLONNKJRba87WAefz8Ca9d6ds/SzrPc235n1IxWYdhJ2zF3MNUaoQ==} peerDependencies: react: ^16.11.0 || ^17.0.0 || ^18.0.0 dependencies: @@ -25867,12 +26141,12 @@ packages: use-sync-external-store: 1.2.0(react@18.2.0) dev: true - /swrev@3.0.0: - resolution: {integrity: sha512-QJuZiptdOmbDY45pECBRVEgnoBlOKjeT2MWVz04wKHpWX15hM3P7EjcIbHDg5yLoPCMQ7to3349MEE+l9QF5HA==} + /swrev@4.0.0: + resolution: {integrity: sha512-LqVcOHSB4cPGgitD1riJ1Hh4vdmITOp+BkmfmXRh4hSF/t7EnS4iD+SOTmq7w5pPm/SiPeto4ADbKS6dHUDWFA==} dev: true - /swrv@1.0.3(vue@3.3.4): - resolution: {integrity: sha512-sl+eLEE+aPPjhP1E8gQ75q3RPRyw5Gd/kROnrTFo3+LkCeLskv7F+uAl5W97wgJkzitobL6FLsRPVm0DgIgN8A==} + /swrv@1.0.4(vue@3.3.4): + resolution: {integrity: sha512-zjEkcP8Ywmj+xOJW3lIT65ciY/4AL4e/Or7Gj0MzU3zBJNMdJiT8geVZhINavnlHRMMCcJLHhraLTAiDOTmQ9g==} peerDependencies: vue: '>=3.2.26 < 4' dependencies: @@ -26399,6 +26673,10 @@ packages: engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} dev: true + /trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + dev: true + /trim-newlines@3.0.1: resolution: {integrity: sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==} engines: {node: '>=8'} @@ -26411,6 +26689,10 @@ packages: escape-string-regexp: 5.0.0 dev: true + /trough@2.1.0: + resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==} + dev: true + /ts-dedent@2.2.0: resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==} engines: {node: '>=6.10'} @@ -26845,6 +27127,18 @@ packages: engines: {node: '>=4'} dev: true + /unified@10.1.2: + resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + dependencies: + '@types/unist': 2.0.6 + bail: 2.0.2 + extend: 3.0.2 + is-buffer: 2.0.5 + is-plain-obj: 4.1.0 + trough: 2.1.0 + vfile: 5.3.7 + dev: true + /union@0.5.0: resolution: {integrity: sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==} engines: {node: '>= 0.8.0'} @@ -26893,6 +27187,10 @@ packages: '@types/unist': 3.0.0 dev: true + /unist-util-generated@2.0.1: + resolution: {integrity: sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==} + dev: true + /unist-util-is@4.1.0: resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==} dev: true @@ -26903,6 +27201,12 @@ packages: '@types/unist': 2.0.6 dev: true + /unist-util-position@4.0.4: + resolution: {integrity: sha512-kUBE91efOWfIVBo8xzh/uZQ7p9ffYRtUbMRZBNFYwf0RK8koUMx6dGUfwylLOKmaT2cs4wSW96QoYUSXAyEtpg==} + dependencies: + '@types/unist': 2.0.6 + dev: true + /unist-util-stringify-position@3.0.3: resolution: {integrity: sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg==} dependencies: @@ -27379,6 +27683,22 @@ packages: extsprintf: 1.4.1 dev: true + /vfile-message@3.1.4: + resolution: {integrity: sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw==} + dependencies: + '@types/unist': 2.0.6 + unist-util-stringify-position: 3.0.3 + dev: true + + /vfile@5.3.7: + resolution: {integrity: sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g==} + dependencies: + '@types/unist': 2.0.6 + is-buffer: 2.0.5 + unist-util-stringify-position: 3.0.3 + vfile-message: 3.1.4 + dev: true + /vite-node@0.32.0(@types/node@18.16.9)(less@4.1.3)(sass@1.55.0)(stylus@0.59.0): resolution: {integrity: sha512-220P/y8YacYAU+daOAqiGEFXx2A8AwjadDzQqos6wSukjvvTWNqleJSwoUn0ckyNdjHIKoxn93Nh1vWBqEKr3Q==} engines: {node: '>=v14.18.0'} @@ -27658,6 +27978,11 @@ packages: setimmediate-napi: 1.0.6 dev: false + /web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + dev: true + /webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} dev: true diff --git a/tools/documentation/create-embeddings/src/main.mts b/tools/documentation/create-embeddings/src/main.mts index c8b873b1c51211..5855fd718152ea 100644 --- a/tools/documentation/create-embeddings/src/main.mts +++ b/tools/documentation/create-embeddings/src/main.mts @@ -6,8 +6,7 @@ import { config as loadDotEnvFile } from 'dotenv'; import { expand } from 'dotenv-expand'; import { readFile } from 'fs/promises'; import 'openai'; -import { Configuration, OpenAIApi } from 'openai'; -import { inspect } from 'util'; +import OpenAI from 'openai'; import yargs from 'yargs'; import { createHash } from 'crypto'; import GithubSlugger from 'github-slugger'; @@ -298,21 +297,15 @@ async function generateEmbeddings() { const input = content.replace(/\n/g, ' '); try { - const configuration = new Configuration({ - apiKey: process.env.NX_OPENAI_KEY, + const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, // This is also the default, can be omitted }); - const openai = new OpenAIApi(configuration); - - const embeddingResponse = await openai.createEmbedding({ + const embeddingResponse = await openai.embeddings.create({ model: 'text-embedding-ada-002', input, }); - if (embeddingResponse.status !== 200) { - throw new Error(inspect(embeddingResponse.data, false, 2)); - } - - const [responseData] = embeddingResponse.data.data; + const [responseData] = embeddingResponse.data; const { error: insertPageSectionError, data: pageSection } = await supabaseClient @@ -323,7 +316,7 @@ async function generateEmbeddings() { heading, content, url_partial, - token_count: embeddingResponse.data.usage.total_tokens, + token_count: embeddingResponse.usage.total_tokens, embedding: responseData.embedding, }) .select() diff --git a/tsconfig.base.json b/tsconfig.base.json index 75742af2184d1d..31757601e7d581 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -47,7 +47,6 @@ "@nx/next/*": ["packages/next/*"], "@nx/node": ["packages/node"], "@nx/node/*": ["packages/node/*"], - "@nx/nx-dev/data-access-ai": ["nx-dev/data-access-ai/src/index.ts"], "@nx/nx-dev/data-access-documents": [ "nx-dev/data-access-documents/src/index.ts" ],