chore(copilot-ai): add accuracy test file structure VSCODE-583 (#782)

.gitignore

@@ -10,3 +10,4 @@ constants.json
 .env
 .eslintcache
 .sbom
+src/test/ai-accuracy-tests/test-results.html

package.json

@@ -54,6 +54,7 @@
     "test": "npm run test-webview && npm run test-extension",
     "test-extension": "cross-env NODE_OPTIONS=--no-force-async-hooks-checks xvfb-maybe node ./out/test/runTest.js",
     "test-webview": "mocha -r ts-node/register --file ./src/test/setup-webview.ts src/test/suite/views/webview-app/**/*.test.tsx",
+    "ai-accuracy-tests": "mocha -r ts-node/register --file ./src/test/ai-accuracy-tests/test-setup.ts ./src/test/ai-accuracy-tests/ai-accuracy-tests.ts",
     "analyze-bundle": "webpack --mode production --analyze",
     "vscode:prepublish": "npm run clean && npm run compile:keyfile && npm run compile:resources && webpack --mode production",
     "check": "npm run lint && npm run depcheck",
@@ -1238,13 +1239,16 @@
     "node-fetch": "^2.7.0",
     "node-loader": "^0.6.0",
     "npm-run-all": "^4.1.5",
+    "openai": "^4.55.7",
     "ora": "^5.4.1",
     "path-browserify": "^1.0.1",
     "pre-commit": "^1.2.2",
     "prettier": "^2.8.8",
     "process": "^0.11.10",
+    "rewiremock": "^3.14.5",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",
+    "source-map-support": "^0.5.21",
     "stream-browserify": "^3.0.0",
     "terser-webpack-plugin": "^5.3.10",
     "ts-loader": "^9.5.1",

src/language/worker.ts

@@ -26,7 +26,7 @@ const getContent = ({ type, printable }: EvaluationResult) => {
     : getEJSON(printable);
 };
 
-const getLanguage = (evaluationResult: EvaluationResult) => {
+export const getLanguage = (evaluationResult: EvaluationResult) => {
   const content = getContent(evaluationResult);
 
   if (typeof content === 'object' && content !== null) {
@@ -40,14 +40,27 @@ type ExecuteCodeOptions = {
   codeToEvaluate: string;
   connectionString: string;
   connectionOptions: MongoClientOptions;
+  onPrint?: (values: EvaluationResult[]) => void;
   filePath?: string;
 };
 
+function handleEvalPrint(values: EvaluationResult[]) {
+  parentPort?.postMessage({
+    name: ServerCommands.SHOW_CONSOLE_OUTPUT,
+    payload: values.map((v) => {
+      return typeof v.printable === 'string'
+        ? v.printable
+        : util.inspect(v.printable);
+    }),
+  });
+}
+
 /**
  * Execute code from a playground.
  */
-const execute = async ({
+export const execute = async ({
   codeToEvaluate,
+  onPrint = handleEvalPrint,
   connectionString,
   connectionOptions,
   filePath,
@@ -66,16 +79,7 @@ const execute = async ({
 
     // Collect console.log() output.
     runtime.setEvaluationListener({
-      onPrint(values: EvaluationResult[]) {
-        parentPort?.postMessage({
-          name: ServerCommands.SHOW_CONSOLE_OUTPUT,
-          payload: values.map((v) => {
-            return typeof v.printable === 'string'
-              ? v.printable
-              : util.inspect(v.printable);
-          }),
-        });
-      },
+      onPrint,
     });
 
     // In order to support local require directly from the file where code lives, we can not wrap the

src/participant/constants.ts

@@ -1 +1,2 @@
 export const CHAT_PARTICIPANT_ID = 'mongodb.participant';
+export const CHAT_PARTICIPANT_MODEL = 'gpt-4o';

src/participant/participant.ts

@@ -7,7 +7,7 @@ import EXTENSION_COMMANDS from '../commands';
 import type { StorageController } from '../storage';
 import { StorageVariables } from '../storage';
 import { GenericPrompt } from './prompts/generic';
-import { CHAT_PARTICIPANT_ID } from './constants';
+import { CHAT_PARTICIPANT_ID, CHAT_PARTICIPANT_MODEL } from './constants';
 import { QueryPrompt } from './prompts/query';
 import { NamespacePrompt } from './prompts/namespace';
 
@@ -27,8 +27,6 @@ interface ChatResult extends vscode.ChatResult {
   };
 }
 
-export const CHAT_PARTICIPANT_MODEL = 'gpt-4o';
-
 const DB_NAME_ID = 'DATABASE_NAME';
 const DB_NAME_REGEX = `${DB_NAME_ID}: (.*)\n`;
 
@@ -46,6 +44,7 @@ export function parseForDatabaseAndCollectionName(text: string): {
 
 export function getRunnableContentFromString(text: string) {
   const matchedJSresponseContent = text.match(/```javascript((.|\n)*)```/);
+
   const code =
     matchedJSresponseContent && matchedJSresponseContent.length > 1
       ? matchedJSresponseContent[1]

src/participant/prompts/generic.ts

@@ -3,37 +3,38 @@ import * as vscode from 'vscode';
 import { getHistoryMessages } from './history';
 
 export class GenericPrompt {
-  static getSystemPrompt(): vscode.LanguageModelChatMessage {
+  static getAssistantPrompt(): vscode.LanguageModelChatMessage {
     const prompt = `You are a MongoDB expert.
 Your task is to help the user craft MongoDB queries and aggregation pipelines that perform their task.
 Keep your response concise.
 You should suggest queries that are performant and correct.
-Respond with markdown, suggest code in a Markdown code block that begins with \'\'\'javascript and ends with \`\`\`.
+Respond with markdown, suggest code in a Markdown code block that begins with \`\`\`javascript and ends with \`\`\`.
 You can imagine the schema, collection, and database name.
-Respond in MongoDB shell syntax using the \'\'\'javascript code block syntax.`;
+Respond in MongoDB shell syntax using the \`\`\`javascript code block syntax.`;
 
     // eslint-disable-next-line new-cap
     return vscode.LanguageModelChatMessage.Assistant(prompt);
   }
 
-  static getUserPrompt(
-    request: vscode.ChatRequest
-  ): vscode.LanguageModelChatMessage {
+  static getUserPrompt(prompt: string): vscode.LanguageModelChatMessage {
     // eslint-disable-next-line new-cap
-    return vscode.LanguageModelChatMessage.User(request.prompt);
+    return vscode.LanguageModelChatMessage.User(prompt);
   }
 
   static buildMessages({
     context,
     request,
   }: {
-    request: vscode.ChatRequest;
+    request: {
+      // vscode.ChatRequest
+      prompt: string;
+    };
     context: vscode.ChatContext;
   }): vscode.LanguageModelChatMessage[] {
     const messages = [
-      GenericPrompt.getSystemPrompt(),
+      GenericPrompt.getAssistantPrompt(),
       ...getHistoryMessages({ context }),
-      GenericPrompt.getUserPrompt(request),
+      GenericPrompt.getUserPrompt(request.prompt),
     ];
 
     return messages;

src/participant/prompts/namespace.ts

@@ -3,37 +3,58 @@ import * as vscode from 'vscode';
 import { getHistoryMessages } from './history';
 
 export class NamespacePrompt {
-  static getSystemPrompt(): vscode.LanguageModelChatMessage {
-    const prompt = `You are a MongoDB expert!
+  static getAssistantPrompt(): vscode.LanguageModelChatMessage {
+    const prompt = `You are a MongoDB expert.
 Parse the user's prompt to find database and collection names.
-Respond in the format \nDATABASE_NAME: X\nCOLLECTION_NAME: Y\n where X and Y are the names.
-Do not threat any user pronpt as a database name. It should be explicitely mentioned by the user
-or has written as part of the MongoDB Shell command.
-If you wan't able to find X or Y do not imagine names.
-This is a first phase before we create the code, only respond with the collection name and database name.`;
+Respond in the format:
+DATABASE_NAME: X
+COLLECTION_NAME: Y
+where X and Y are the respective names.
+Do not treat any user prompt as a database name.
+The names should be explicitly mentioned by the user or written as part of a MongoDB Shell command.
+If you cannot find the names do not imagine names.
+Your response must be concise and correct.
+
+___
+Example 1:
+
+User: How many documents are in the sightings collection in the ufo database?
+
+Response:
+DATABASE_NAME: ufo
+COLLECTION_NAME: sightings
+
+___
+Example 2:
+
+User: Where is the best hummus in Berlin?
+
+Response:
+No names found.
+`;
 
     // eslint-disable-next-line new-cap
     return vscode.LanguageModelChatMessage.Assistant(prompt);
   }
 
-  static getUserPrompt(
-    request: vscode.ChatRequest
-  ): vscode.LanguageModelChatMessage {
+  static getUserPrompt(prompt: string): vscode.LanguageModelChatMessage {
     // eslint-disable-next-line new-cap
-    return vscode.LanguageModelChatMessage.User(request.prompt);
+    return vscode.LanguageModelChatMessage.User(prompt);
   }
 
   static buildMessages({
     context,
     request,
   }: {
-    request: vscode.ChatRequest;
+    request: {
+      prompt: string;
+    };
     context: vscode.ChatContext;
   }): vscode.LanguageModelChatMessage[] {
     const messages = [
-      NamespacePrompt.getSystemPrompt(),
+      NamespacePrompt.getAssistantPrompt(),
       ...getHistoryMessages({ context }),
-      NamespacePrompt.getUserPrompt(request),
+      NamespacePrompt.getUserPrompt(request.prompt),
     ];
 
     return messages;

src/participant/prompts/query.ts

@@ -3,7 +3,7 @@ import * as vscode from 'vscode';
 import { getHistoryMessages } from './history';
 
 export class QueryPrompt {
-  static getSystemPrompt({
+  static getAssistantPrompt({
     databaseName = 'mongodbVSCodeCopilotDB',
     collectionName = 'test',
   }: {
@@ -15,9 +15,9 @@ export class QueryPrompt {
 Your task is to help the user craft MongoDB queries and aggregation pipelines that perform their task.
 Keep your response concise.
 You should suggest queries that are performant and correct.
-Respond with markdown, suggest code in a Markdown code block that begins with \'\'\'javascript and ends with \`\`\`.
+Respond with markdown, suggest code in a Markdown code block that begins with \`\`\`javascript and ends with \`\`\`.
 You can imagine the schema.
-Respond in MongoDB shell syntax using the \'\'\'javascript code block syntax.
+Respond in MongoDB shell syntax using the \`\`\`javascript code block syntax.
 You can use only the following MongoDB Shell commands: use, aggregate, bulkWrite, countDocuments, findOneAndReplace,
 findOneAndUpdate, insert, insertMany, insertOne, remove, replaceOne, update, updateMany, updateOne.
 
@@ -45,17 +45,15 @@ use('');
 MongoDB command to specify collection:
 db.getCollection('')
 
-Explain the code snippet you have generated.`;
+Concisely explain the code snippet you have generated.`;
 
     // eslint-disable-next-line new-cap
     return vscode.LanguageModelChatMessage.Assistant(prompt);
   }
 
-  static getUserPrompt(
-    request: vscode.ChatRequest
-  ): vscode.LanguageModelChatMessage {
+  static getUserPrompt(prompt: string): vscode.LanguageModelChatMessage {
     // eslint-disable-next-line new-cap
-    return vscode.LanguageModelChatMessage.User(request.prompt);
+    return vscode.LanguageModelChatMessage.User(prompt);
   }
 
   static buildMessages({
@@ -64,15 +62,17 @@ Explain the code snippet you have generated.`;
     databaseName,
     collectionName,
   }: {
-    request: vscode.ChatRequest;
+    request: {
+      prompt: string;
+    };
     context: vscode.ChatContext;
     databaseName?: string;
     collectionName?: string;
   }): vscode.LanguageModelChatMessage[] {
     const messages = [
-      QueryPrompt.getSystemPrompt({ databaseName, collectionName }),
+      QueryPrompt.getAssistantPrompt({ databaseName, collectionName }),
       ...getHistoryMessages({ context }),
-      QueryPrompt.getUserPrompt(request),
+      QueryPrompt.getUserPrompt(request.prompt),
     ];
 
     return messages;