From 5f5d9e9a4972da6e09caa0d96bcf8cc3b4ff0e24 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Wed, 25 Sep 2024 21:32:25 +0200 Subject: [PATCH 1/6] Wire up telemetry for user prompts --- src/participant/participant.ts | 27 ++-- src/participant/prompts/promptBase.ts | 56 ++++++++- src/participant/prompts/query.ts | 20 +-- src/participant/prompts/schema.ts | 15 ++- src/telemetry/telemetryService.ts | 12 ++ .../ai-accuracy-tests/ai-accuracy-tests.ts | 7 +- .../suite/participant/participant.test.ts | 118 +++++++++++++++--- 7 files changed, 204 insertions(+), 51 deletions(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index dc9e94a17..0a680dc2e 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -38,6 +38,7 @@ import { } from '../telemetry/telemetryService'; import { DocsChatbotAIService } from './docsChatbotAIService'; import type TelemetryService from '../telemetry/telemetryService'; +import type { PromptResult } from './prompts/promptBase'; const log = createLogger('participant'); @@ -171,19 +172,21 @@ export default class ParticipantController { } async getChatResponseContent({ - messages, + prompt, token, }: { - messages: vscode.LanguageModelChatMessage[]; + prompt: PromptResult; token: vscode.CancellationToken; }): Promise { const model = await getCopilotModel(); let responseContent = ''; if (model) { - const chatResponse = await model.sendRequest(messages, {}, token); + const chatResponse = await model.sendRequest(prompt.messages, {}, token); for await (const fragment of chatResponse.text) { responseContent += fragment; } + + this._telemetryService.trackCopilotParticipantPrompt(prompt.stats); } return responseContent; @@ -221,14 +224,14 @@ export default class ParticipantController { stream: vscode.ChatResponseStream, token: vscode.CancellationToken ): Promise { - const messages = await Prompts.generic.buildMessages({ + const prompt = await Prompts.generic.buildMessages({ request, context, connectionNames: this._getConnectionNames(), }); const responseContent = await this.getChatResponseContent({ - messages, + prompt, token, }); stream.markdown(responseContent); @@ -569,7 +572,7 @@ export default class ParticipantController { connectionNames: this._getConnectionNames(), }); const responseContentWithNamespace = await this.getChatResponseContent({ - messages: messagesWithNamespace, + prompt: messagesWithNamespace, token, }); const { databaseName, collectionName } = @@ -904,7 +907,7 @@ export default class ParticipantController { return schemaRequestChatResult(context.history); } - const messages = await Prompts.schema.buildMessages({ + const prompt = await Prompts.schema.buildMessages({ request, context, databaseName, @@ -915,7 +918,7 @@ export default class ParticipantController { ...(sampleDocuments ? { sampleDocuments } : {}), }); const responseContent = await this.getChatResponseContent({ - messages, + prompt, token, }); stream.markdown(responseContent); @@ -1008,7 +1011,7 @@ export default class ParticipantController { ); } - const messages = await Prompts.query.buildMessages({ + const prompt = await Prompts.query.buildMessages({ request, context, databaseName, @@ -1018,7 +1021,7 @@ export default class ParticipantController { ...(sampleDocuments ? { sampleDocuments } : {}), }); const responseContent = await this.getChatResponseContent({ - messages, + prompt, token, }); @@ -1099,14 +1102,14 @@ export default class ParticipantController { responseReferences?: Reference[]; }> { const [request, context, , token] = args; - const messages = await Prompts.generic.buildMessages({ + const prompt = await Prompts.generic.buildMessages({ request, context, connectionNames: this._getConnectionNames(), }); const responseContent = await this.getChatResponseContent({ - messages, + prompt, token, }); const responseReferences = [ diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index 0f3c83286..cf7a4cf14 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import type { ChatResult, ParticipantResponseType } from '../constants'; +import type { ParticipantPromptProperties } from '../../telemetry/telemetryService'; export interface PromptArgsBase { request: { @@ -10,14 +11,37 @@ export interface PromptArgsBase { connectionNames: string[]; } +export interface UserPromptResponse { + prompt: string; + hasSampleDocs: boolean; +} + +export interface PromptResult { + messages: vscode.LanguageModelChatMessage[]; + stats: ParticipantPromptProperties; +} + +export interface UserPromptResponse { + prompt: string; + hasSampleDocs: boolean; +} + +export interface PromptResult { + messages: vscode.LanguageModelChatMessage[]; + stats: ParticipantPromptProperties; +} + export abstract class PromptBase { protected abstract getAssistantPrompt(args: TArgs): string; - protected getUserPrompt(args: TArgs): Promise { - return Promise.resolve(args.request.prompt); + protected getUserPrompt(args: TArgs): Promise { + return Promise.resolve({ + prompt: args.request.prompt, + hasSampleDocs: false, + }); } - async buildMessages(args: TArgs): Promise { + async buildMessages(args: TArgs): Promise { let historyMessages = this.getHistoryMessages(args); // If the current user's prompt is a connection name, and the last // message was to connect. We want to use the last @@ -49,13 +73,35 @@ export abstract class PromptBase { } } - return [ + const { prompt, hasSampleDocs } = await this.getUserPrompt(args); + const messages = [ // eslint-disable-next-line new-cap vscode.LanguageModelChatMessage.Assistant(this.getAssistantPrompt(args)), ...historyMessages, // eslint-disable-next-line new-cap - vscode.LanguageModelChatMessage.User(await this.getUserPrompt(args)), + vscode.LanguageModelChatMessage.User(prompt), ]; + + return { + messages, + stats: this.getStats(messages, args, hasSampleDocs), + }; + } + + protected getStats( + messages: vscode.LanguageModelChatMessage[], + { request }: TArgs, + hasSampleDocs: boolean + ): ParticipantPromptProperties { + return { + total_message_length: messages.reduce( + (acc, message) => acc + message.content.length, + 0 + ), + user_input_length: request.prompt.length, + has_sample_documents: hasSampleDocs, + command: request.command || 'generic', + }; } // When passing the history to the model we only want contextual messages diff --git a/src/participant/prompts/query.ts b/src/participant/prompts/query.ts index b7ae5cc26..926b739c8 100644 --- a/src/participant/prompts/query.ts +++ b/src/participant/prompts/query.ts @@ -2,7 +2,7 @@ import * as vscode from 'vscode'; import type { Document } from 'bson'; import { getStringifiedSampleDocuments } from '../sampleDocuments'; -import type { PromptArgsBase } from './promptBase'; +import type { PromptArgsBase, UserPromptResponse } from './promptBase'; import { PromptBase } from './promptBase'; interface QueryPromptArgs extends PromptArgsBase { @@ -58,21 +58,23 @@ db.getCollection('');\n`; request, schema, sampleDocuments, - }: QueryPromptArgs): Promise { + }: QueryPromptArgs): Promise { let prompt = request.prompt; prompt += `\nDatabase name: ${databaseName}\n`; prompt += `Collection name: ${collectionName}\n`; if (schema) { prompt += `Collection schema: ${schema}\n`; } - if (sampleDocuments) { - prompt += await getStringifiedSampleDocuments({ - sampleDocuments, - prompt, - }); - } - return prompt; + const sampleDocumentsPrompt = await getStringifiedSampleDocuments({ + sampleDocuments, + prompt, + }); + + return { + prompt: `${prompt}${sampleDocumentsPrompt}`, + hasSampleDocs: !!sampleDocumentsPrompt, + }; } get emptyRequestResponse(): string { diff --git a/src/participant/prompts/schema.ts b/src/participant/prompts/schema.ts index 895f99568..ca8b54b26 100644 --- a/src/participant/prompts/schema.ts +++ b/src/participant/prompts/schema.ts @@ -1,3 +1,4 @@ +import type { UserPromptResponse } from './promptBase'; import { PromptBase, type PromptArgsBase } from './promptBase'; export const DOCUMENTS_TO_SAMPLE_FOR_SCHEMA_PROMPT = 100; @@ -11,7 +12,6 @@ export interface SchemaPromptArgs extends PromptArgsBase { collectionName: string; schema: string; amountOfDocumentsSampled: number; - connectionNames: string[]; } export class SchemaPrompt extends PromptBase { @@ -30,13 +30,16 @@ Amount of documents sampled: ${amountOfDocumentsSampled}.`; collectionName, request, schema, - }: SchemaPromptArgs): Promise { + }: SchemaPromptArgs): Promise { const prompt = request.prompt; - return Promise.resolve(`${ - prompt ? `The user provided additional information: "${prompt}"\n` : '' - }Database name: ${databaseName} + return Promise.resolve({ + prompt: `${ + prompt ? `The user provided additional information: "${prompt}"\n` : '' + }Database name: ${databaseName} Collection name: ${collectionName} Schema: -${schema}`); +${schema}`, + hasSampleDocs: false, + }); } } diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 53dd2cba5..2f2e770bc 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -108,6 +108,13 @@ type ParticipantResponseFailedProperties = { error_name: ParticipantErrorTypes; }; +export type ParticipantPromptProperties = { + command: string; + user_input_length: number; + total_message_length: number; + has_sample_documents: boolean; +}; + export function chatResultFeedbackKindToTelemetryValue( kind: vscode.ChatResultFeedbackKind ): TelemetryFeedbackKind { @@ -160,6 +167,7 @@ export enum TelemetryEventTypes { PARTICIPANT_FEEDBACK = 'Participant Feedback', PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', + PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', } export enum ParticipantErrorTypes { @@ -422,4 +430,8 @@ export default class TelemetryService { trackCopilotParticipantFeedback(props: ParticipantFeedbackProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_FEEDBACK, props); } + + trackCopilotParticipantPrompt(stats: ParticipantPromptProperties): void { + this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats); + } } diff --git a/src/test/ai-accuracy-tests/ai-accuracy-tests.ts b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts index 77db0244b..7f0957273 100644 --- a/src/test/ai-accuracy-tests/ai-accuracy-tests.ts +++ b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts @@ -19,6 +19,7 @@ import { } from './create-test-results-html-page'; import { runCodeInMessage } from './assertions'; import { Prompts } from '../../participant/prompts'; +import type { PromptResult } from '../../participant/prompts/promptBase'; const numberOfRunsPerTest = 1; @@ -317,10 +318,10 @@ const buildMessages = async ({ }: { testCase: TestCase; fixtures: Fixtures; -}): Promise => { +}): Promise => { switch (testCase.type) { case 'generic': - return Prompts.generic.buildMessages({ + return await Prompts.generic.buildMessages({ request: { prompt: testCase.userInput }, context: { history: [] }, connectionNames: [], @@ -373,7 +374,7 @@ async function runTest({ aiBackend: AIBackend; fixtures: Fixtures; }): Promise { - const messages = await buildMessages({ + const { messages } = await buildMessages({ testCase, fixtures, }); diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index d52debb46..87ba27ab4 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -13,6 +13,8 @@ import ConnectionController from '../../../connectionController'; import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; import { ExtensionContextStub } from '../stubs'; +import type { + ParticipantPromptProperties } from '../../../telemetry/telemetryService'; import TelemetryService, { TelemetryEventTypes, } from '../../../telemetry/telemetryService'; @@ -47,7 +49,7 @@ const encodeStringify = (obj: Record): string => { return encodeURIComponent(JSON.stringify(obj)); }; -suite('Participant Controller Test Suite', function () { +suite.only('Participant Controller Test Suite', function () { const extensionContextStub = new ExtensionContextStub(); // The test extension runner. @@ -79,6 +81,23 @@ suite('Participant Controller Test Suite', function () { chatTokenStub ); + const assertCommandTelemetry = (command: string, chatRequest: vscode.ChatRequest, { expectSampleDocs = false }): void => { + expect(telemetryTrackStub).to.have.called; + expect(telemetryTrackStub.lastCall.args[0]).to.equal('Participant Prompt Submitted'); + + const properties = telemetryTrackStub.lastCall.args[1] as ParticipantPromptProperties; + + expect(properties.command).to.equal(command); + expect(properties.has_sample_documents).to.equal(expectSampleDocs); + + // Total message length includes participant as well as user prompt + expect(properties.total_message_length).to.be.greaterThan(properties.user_input_length); + + // User prompt length should be at least equal to the supplied user prompt, but my occasionally + // be greater - e.g. when we enhance the context. + expect(properties.user_input_length).to.be.greaterThanOrEqual(chatRequest.prompt.length); + }; + beforeEach(function () { testStorageController = new StorageController(extensionContextStub); testStatusView = new StatusView(extensionContextStub); @@ -396,11 +415,13 @@ suite('Participant Controller Test Suite', function () { const welcomeMessage = chatStreamStub.markdown.firstCall.args[0]; expect(welcomeMessage).to.include('Welcome to MongoDB Participant!'); - sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.equal( + // Once to report welcome screen shown, second time to track the user prompt + expect(telemetryTrackStub).to.have.been.calledTwice; + expect(telemetryTrackStub.firstCall.args[0]).to.equal( TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN ); - expect(telemetryTrackStub.lastCall.args[1]).to.be.undefined; + expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); }); }); @@ -455,6 +476,8 @@ suite('Participant Controller Test Suite', function () { }, ], }); + + assertCommandTelemetry('generic', chatRequestMock, { expectSampleDocs: false }); }); }); @@ -483,6 +506,8 @@ suite('Participant Controller Test Suite', function () { }, ], }); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); }); test('includes a collection schema', async function () { @@ -508,6 +533,8 @@ suite('Participant Controller Test Suite', function () { 'field.stringField: String\n' + 'field.arrayField: Array\n' ); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); }); suite('useSampleDocsInCopilot setting is true', function () { @@ -574,6 +601,8 @@ suite('Participant Controller Test Suite', function () { ' }\n' + ']\n' ); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); }); test('includes 1 sample document as an object', async function () { @@ -618,6 +647,8 @@ suite('Participant Controller Test Suite', function () { ' }\n' + '}\n' ); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); }); test('includes 1 sample documents when 3 make prompt too long', async function () { @@ -660,6 +691,8 @@ suite('Participant Controller Test Suite', function () { ' }\n' + '}\n' ); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); }); test('does not include sample documents when even 1 makes prompt too long', async function () { @@ -697,6 +730,8 @@ suite('Participant Controller Test Suite', function () { await invokeChatHandler(chatRequestMock); const messages = sendRequestStub.secondCall.args[0]; expect(messages[1].content).to.not.include('Sample documents'); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); }); }); @@ -710,6 +745,8 @@ suite('Participant Controller Test Suite', function () { await invokeChatHandler(chatRequestMock); const messages = sendRequestStub.secondCall.args[0]; expect(messages[1].content).to.not.include('Sample documents'); + + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); }); }); }); @@ -1271,12 +1308,12 @@ Schema: expect(sendRequestStub).to.have.been.called; // Expect the error to be reported through the telemetry service - sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.equal( + expect(telemetryTrackStub).to.have.been.calledTwice; + expect(telemetryTrackStub.firstCall.args[0]).to.equal( TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED ); - const properties = telemetryTrackStub.lastCall.args[1]; + const properties = telemetryTrackStub.firstCall.args[1]; expect(properties.command).to.equal('docs'); expect(properties.error_name).to.equal('Docs Chatbot API Issue'); }); @@ -1289,7 +1326,7 @@ Schema: const chatRequestMock = { prompt: 'find all docs by a name example', }; - const messages = await Prompts.generic.buildMessages({ + const { messages, stats } = await Prompts.generic.buildMessages({ context: chatContextStub, request: chatRequestMock, connectionNames: [], @@ -1302,6 +1339,13 @@ Schema: expect(messages[1].role).to.equal( vscode.LanguageModelChatMessageRole.User ); + + expect(stats.command).to.equal('generic'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); }); test('query', async function () { @@ -1321,7 +1365,7 @@ Schema: }), ], }; - const messages = await Prompts.query.buildMessages({ + const { messages, stats } = await Prompts.query.buildMessages({ context: chatContextStub, request: chatRequestMock, collectionName: 'people', @@ -1364,6 +1408,21 @@ Schema: expect(messages[2].role).to.equal( vscode.LanguageModelChatMessageRole.User ); + + expect(stats.command).to.equal('query'); + expect(stats.has_sample_documents).to.be.true; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + + messages[1].content.length + + messages[2].content.length + ); + + // The actual user prompt length should be the prompt supplied by the user, even if + // we enhance it with sample docs and schema. + expect(stats.user_input_length).to.be.lessThan( + messages[2].content.length + ); }); test('schema', async function () { @@ -1380,7 +1439,7 @@ Schema: name: String } `; - const messages = await Prompts.schema.buildMessages({ + const { messages, stats } = await Prompts.schema.buildMessages({ context: chatContextStub, request: chatRequestMock, amountOfDocumentsSampled: 3, @@ -1402,14 +1461,21 @@ Schema: expect(messages[1].content).to.include(databaseName); expect(messages[1].content).to.include(collectionName); expect(messages[1].content).to.include(schema); + + expect(stats.command).to.equal('schema'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); }); test('namespace', async function () { const chatRequestMock = { prompt: 'find all docs by a name example', - command: 'query', + command: 'schema', }; - const messages = await Prompts.namespace.buildMessages({ + const { messages, stats } = await Prompts.namespace.buildMessages({ context: chatContextStub, request: chatRequestMock, connectionNames: [], @@ -1422,6 +1488,13 @@ Schema: expect(messages[1].role).to.equal( vscode.LanguageModelChatMessageRole.User ); + + expect(stats.command).to.equal('schema'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); }); test('removes askForConnect messages from history', async function () { @@ -1432,10 +1505,13 @@ Schema: command: 'query', }; + // This is the prompt of the user prior to us asking them to connect + const expectedPrompt = 'give me the count of all people in the prod database'; + chatContextStub = { history: [ Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { - prompt: 'give me the count of all people in the prod database', + prompt: expectedPrompt, command: 'query', references: [], participant: CHAT_PARTICIPANT_ID, @@ -1471,7 +1547,7 @@ Schema: ], }; - const messages = await Prompts.query.buildMessages({ + const { messages, stats } = await Prompts.query.buildMessages({ context: chatContextStub, request: chatRequestMock, collectionName: 'people', @@ -1491,8 +1567,18 @@ Schema: expect(messages[1].role).to.equal( vscode.LanguageModelChatMessageRole.User ); - expect(messages[1].content).to.contain( - 'give me the count of all people in the prod database' + expect(messages[1].content).to.contain(expectedPrompt); + + expect(stats.command).to.equal('query'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(expectedPrompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); + + // The prompt builder may add extra info, but we're only reporting the actual user input + expect(stats.user_input_length).to.be.lessThan( + messages[1].content.length ); }); }); From 24d53783fb3d9061d83367246ef41e61044bff58 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Thu, 26 Sep 2024 14:02:30 +0200 Subject: [PATCH 2/6] Remove .only --- .../suite/participant/participant.test.ts | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 87ba27ab4..5c1df7504 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -13,8 +13,7 @@ import ConnectionController from '../../../connectionController'; import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; import { ExtensionContextStub } from '../stubs'; -import type { - ParticipantPromptProperties } from '../../../telemetry/telemetryService'; +import type { ParticipantPromptProperties } from '../../../telemetry/telemetryService'; import TelemetryService, { TelemetryEventTypes, } from '../../../telemetry/telemetryService'; @@ -49,7 +48,7 @@ const encodeStringify = (obj: Record): string => { return encodeURIComponent(JSON.stringify(obj)); }; -suite.only('Participant Controller Test Suite', function () { +suite('Participant Controller Test Suite', function () { const extensionContextStub = new ExtensionContextStub(); // The test extension runner. @@ -81,21 +80,32 @@ suite.only('Participant Controller Test Suite', function () { chatTokenStub ); - const assertCommandTelemetry = (command: string, chatRequest: vscode.ChatRequest, { expectSampleDocs = false }): void => { + const assertCommandTelemetry = ( + command: string, + chatRequest: vscode.ChatRequest, + { expectSampleDocs = false } + ): void => { expect(telemetryTrackStub).to.have.called; - expect(telemetryTrackStub.lastCall.args[0]).to.equal('Participant Prompt Submitted'); + expect(telemetryTrackStub.lastCall.args[0]).to.equal( + 'Participant Prompt Submitted' + ); - const properties = telemetryTrackStub.lastCall.args[1] as ParticipantPromptProperties; + const properties = telemetryTrackStub.lastCall + .args[1] as ParticipantPromptProperties; expect(properties.command).to.equal(command); expect(properties.has_sample_documents).to.equal(expectSampleDocs); // Total message length includes participant as well as user prompt - expect(properties.total_message_length).to.be.greaterThan(properties.user_input_length); + expect(properties.total_message_length).to.be.greaterThan( + properties.user_input_length + ); // User prompt length should be at least equal to the supplied user prompt, but my occasionally // be greater - e.g. when we enhance the context. - expect(properties.user_input_length).to.be.greaterThanOrEqual(chatRequest.prompt.length); + expect(properties.user_input_length).to.be.greaterThanOrEqual( + chatRequest.prompt.length + ); }; beforeEach(function () { @@ -421,7 +431,9 @@ suite.only('Participant Controller Test Suite', function () { TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN ); expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: false, + }); }); }); @@ -477,7 +489,9 @@ suite.only('Participant Controller Test Suite', function () { ], }); - assertCommandTelemetry('generic', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('generic', chatRequestMock, { + expectSampleDocs: false, + }); }); }); @@ -507,7 +521,9 @@ suite.only('Participant Controller Test Suite', function () { ], }); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: false, + }); }); test('includes a collection schema', async function () { @@ -534,7 +550,9 @@ suite.only('Participant Controller Test Suite', function () { 'field.arrayField: Array\n' ); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: false, + }); }); suite('useSampleDocsInCopilot setting is true', function () { @@ -602,7 +620,9 @@ suite.only('Participant Controller Test Suite', function () { ']\n' ); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + }); }); test('includes 1 sample document as an object', async function () { @@ -648,7 +668,9 @@ suite.only('Participant Controller Test Suite', function () { '}\n' ); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + }); }); test('includes 1 sample documents when 3 make prompt too long', async function () { @@ -692,7 +714,9 @@ suite.only('Participant Controller Test Suite', function () { '}\n' ); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + }); }); test('does not include sample documents when even 1 makes prompt too long', async function () { @@ -731,7 +755,9 @@ suite.only('Participant Controller Test Suite', function () { const messages = sendRequestStub.secondCall.args[0]; expect(messages[1].content).to.not.include('Sample documents'); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: false, + }); }); }); @@ -746,7 +772,9 @@ suite.only('Participant Controller Test Suite', function () { const messages = sendRequestStub.secondCall.args[0]; expect(messages[1].content).to.not.include('Sample documents'); - assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: false }); + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: false, + }); }); }); }); @@ -1506,7 +1534,8 @@ Schema: }; // This is the prompt of the user prior to us asking them to connect - const expectedPrompt = 'give me the count of all people in the prod database'; + const expectedPrompt = + 'give me the count of all people in the prod database'; chatContextStub = { history: [ From cde93d6ba1c239146e413eca707b69c559f26b19 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Thu, 26 Sep 2024 14:10:26 +0200 Subject: [PATCH 3/6] Clean up rebase artifacts --- src/participant/prompts/promptBase.ts | 10 ---------- src/test/suite/participant/participant.test.ts | 10 +++++----- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index cf7a4cf14..4933ab65a 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -21,16 +21,6 @@ export interface PromptResult { stats: ParticipantPromptProperties; } -export interface UserPromptResponse { - prompt: string; - hasSampleDocs: boolean; -} - -export interface PromptResult { - messages: vscode.LanguageModelChatMessage[]; - stats: ParticipantPromptProperties; -} - export abstract class PromptBase { protected abstract getAssistantPrompt(args: TArgs): string; diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 5c1df7504..a6465020a 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1446,8 +1446,8 @@ Schema: messages[2].content.length ); - // The actual user prompt length should be the prompt supplied by the user, even if - // we enhance it with sample docs and schema. + // The length of the user prompt length should be taken from the prompt supplied + // by the user, even if we enhance it with sample docs and schema. expect(stats.user_input_length).to.be.lessThan( messages[2].content.length ); @@ -1498,10 +1498,10 @@ Schema: ); }); - test('namespace', async function () { + test.only('namespace', async function () { const chatRequestMock = { prompt: 'find all docs by a name example', - command: 'schema', + command: 'query', }; const { messages, stats } = await Prompts.namespace.buildMessages({ context: chatContextStub, @@ -1517,7 +1517,7 @@ Schema: vscode.LanguageModelChatMessageRole.User ); - expect(stats.command).to.equal('schema'); + expect(stats.command).to.equal('query'); expect(stats.has_sample_documents).to.be.false; expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); expect(stats.total_message_length).to.equal( From af40e40e6f148d7717c8d559d174b9b9060affe0 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Thu, 26 Sep 2024 14:12:02 +0200 Subject: [PATCH 4/6] Remove .only --- src/test/suite/participant/participant.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index a6465020a..c5e632aed 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1498,7 +1498,7 @@ Schema: ); }); - test.only('namespace', async function () { + test('namespace', async function () { const chatRequestMock = { prompt: 'find all docs by a name example', command: 'query', From afa2a33d1f95fecc2f37cdf12e16de2f5ee343c5 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Fri, 27 Sep 2024 22:06:37 +0200 Subject: [PATCH 5/6] Address CR feedback --- src/participant/participant.ts | 50 ++++----- src/participant/prompts/intent.ts | 5 + src/participant/prompts/namespace.ts | 5 + src/participant/prompts/promptBase.ts | 17 ++- src/telemetry/telemetryService.ts | 4 + .../ai-accuracy-tests/ai-accuracy-tests.ts | 4 +- .../suite/participant/participant.test.ts | 103 +++++++++++++++--- 7 files changed, 143 insertions(+), 45 deletions(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 14f50d31b..0d7962af7 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -39,7 +39,7 @@ import { } from '../telemetry/telemetryService'; import { DocsChatbotAIService } from './docsChatbotAIService'; import type TelemetryService from '../telemetry/telemetryService'; -import type { PromptResult } from './prompts/promptBase'; +import type { ModelInput } from './prompts/promptBase'; import { processStreamWithIdentifiers } from './streamParsing'; import type { PromptIntent } from './prompts/intent'; @@ -165,10 +165,10 @@ export default class ParticipantController { } async _getChatResponse({ - prompt, + modelInput, token, }: { - prompt: PromptResult; + modelInput: ModelInput; token: vscode.CancellationToken; }): Promise { const model = await getCopilotModel(); @@ -177,22 +177,22 @@ export default class ParticipantController { throw new Error('Copilot model not found'); } - this._telemetryService.trackCopilotParticipantPrompt(prompt.stats); + this._telemetryService.trackCopilotParticipantPrompt(modelInput.stats); - return await model.sendRequest(prompt.messages, {}, token); + return await model.sendRequest(modelInput.messages, {}, token); } async streamChatResponse({ - prompt, + modelInput, stream, token, }: { - prompt: PromptResult; + modelInput: ModelInput; stream: vscode.ChatResponseStream; token: vscode.CancellationToken; }): Promise { const chatResponse = await this._getChatResponse({ - prompt, + modelInput, token, }); for await (const fragment of chatResponse.text) { @@ -229,16 +229,16 @@ export default class ParticipantController { } async streamChatResponseContentWithCodeActions({ - prompt, + modelInput, stream, token, }: { - prompt: PromptResult; + modelInput: ModelInput; stream: vscode.ChatResponseStream; token: vscode.CancellationToken; }): Promise { const chatResponse = await this._getChatResponse({ - prompt, + modelInput, token, }); @@ -257,15 +257,15 @@ export default class ParticipantController { // This will stream all of the response content and create a string from it. // It should only be used when the entire response is needed at one time. async getChatResponseContent({ - prompt, + modelInput, token, }: { - prompt: PromptResult; + modelInput: ModelInput; token: vscode.CancellationToken; }): Promise { let responseContent = ''; const chatResponse = await this._getChatResponse({ - prompt, + modelInput, token, }); for await (const fragment of chatResponse.text) { @@ -281,14 +281,14 @@ export default class ParticipantController { stream: vscode.ChatResponseStream, token: vscode.CancellationToken ): Promise { - const prompt = await Prompts.generic.buildMessages({ + const modelInput = await Prompts.generic.buildMessages({ request, context, connectionNames: this._getConnectionNames(), }); await this.streamChatResponseContentWithCodeActions({ - prompt, + modelInput, token, stream, }); @@ -337,14 +337,14 @@ export default class ParticipantController { request: vscode.ChatRequest; token: vscode.CancellationToken; }): Promise { - const prompt = await Prompts.intent.buildMessages({ + const modelInput = await Prompts.intent.buildMessages({ connectionNames: this._getConnectionNames(), request, context, }); const responseContent = await this.getChatResponseContent({ - prompt, + modelInput, token, }); @@ -711,7 +711,7 @@ export default class ParticipantController { connectionNames: this._getConnectionNames(), }); const responseContentWithNamespace = await this.getChatResponseContent({ - prompt: messagesWithNamespace, + modelInput: messagesWithNamespace, token, }); const { databaseName, collectionName } = @@ -1046,7 +1046,7 @@ export default class ParticipantController { return schemaRequestChatResult(context.history); } - const prompt = await Prompts.schema.buildMessages({ + const modelInput = await Prompts.schema.buildMessages({ request, context, databaseName, @@ -1057,7 +1057,7 @@ export default class ParticipantController { ...(sampleDocuments ? { sampleDocuments } : {}), }); await this.streamChatResponse({ - prompt, + modelInput, stream, token, }); @@ -1150,7 +1150,7 @@ export default class ParticipantController { ); } - const prompt = await Prompts.query.buildMessages({ + const modelInput = await Prompts.query.buildMessages({ request, context, databaseName, @@ -1161,7 +1161,7 @@ export default class ParticipantController { }); await this.streamChatResponseContentWithCodeActions({ - prompt, + modelInput, stream, token, }); @@ -1233,14 +1233,14 @@ export default class ParticipantController { ] ): Promise { const [request, context, stream, token] = args; - const prompt = await Prompts.generic.buildMessages({ + const modelInput = await Prompts.generic.buildMessages({ request, context, connectionNames: this._getConnectionNames(), }); await this.streamChatResponseContentWithCodeActions({ - prompt, + modelInput, stream, token, }); diff --git a/src/participant/prompts/intent.ts b/src/participant/prompts/intent.ts index 4d6216afa..8a1266f69 100644 --- a/src/participant/prompts/intent.ts +++ b/src/participant/prompts/intent.ts @@ -1,3 +1,4 @@ +import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; import type { PromptArgsBase } from './promptBase'; import { PromptBase } from './promptBase'; @@ -47,4 +48,8 @@ Docs`; return 'Default'; } } + + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return 'intent'; + } } diff --git a/src/participant/prompts/namespace.ts b/src/participant/prompts/namespace.ts index e29f24d2c..c5428f191 100644 --- a/src/participant/prompts/namespace.ts +++ b/src/participant/prompts/namespace.ts @@ -1,3 +1,4 @@ +import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; import type { PromptArgsBase } from './promptBase'; import { PromptBase } from './promptBase'; @@ -50,4 +51,8 @@ No names found. const collectionName = text.match(COL_NAME_REGEX)?.[1].trim(); return { databaseName, collectionName }; } + + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return 'namespace'; + } } diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index 4933ab65a..2885be48d 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -1,6 +1,9 @@ import * as vscode from 'vscode'; import type { ChatResult, ParticipantResponseType } from '../constants'; -import type { ParticipantPromptProperties } from '../../telemetry/telemetryService'; +import type { + InternalPromptPurpose, + ParticipantPromptProperties, +} from '../../telemetry/telemetryService'; export interface PromptArgsBase { request: { @@ -16,7 +19,7 @@ export interface UserPromptResponse { hasSampleDocs: boolean; } -export interface PromptResult { +export interface ModelInput { messages: vscode.LanguageModelChatMessage[]; stats: ParticipantPromptProperties; } @@ -24,6 +27,10 @@ export interface PromptResult { export abstract class PromptBase { protected abstract getAssistantPrompt(args: TArgs): string; + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return undefined; + } + protected getUserPrompt(args: TArgs): Promise { return Promise.resolve({ prompt: args.request.prompt, @@ -31,7 +38,7 @@ export abstract class PromptBase { }); } - async buildMessages(args: TArgs): Promise { + async buildMessages(args: TArgs): Promise { let historyMessages = this.getHistoryMessages(args); // If the current user's prompt is a connection name, and the last // message was to connect. We want to use the last @@ -80,7 +87,7 @@ export abstract class PromptBase { protected getStats( messages: vscode.LanguageModelChatMessage[], - { request }: TArgs, + { request, context }: TArgs, hasSampleDocs: boolean ): ParticipantPromptProperties { return { @@ -91,6 +98,8 @@ export abstract class PromptBase { user_input_length: request.prompt.length, has_sample_documents: hasSampleDocs, command: request.command || 'generic', + history_length: context.history.length, + internal_purpose: this.internalPurposeForTelemetry, }; } diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 2f2e770bc..bfbc200d6 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -108,11 +108,15 @@ type ParticipantResponseFailedProperties = { error_name: ParticipantErrorTypes; }; +export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; + export type ParticipantPromptProperties = { command: string; user_input_length: number; total_message_length: number; has_sample_documents: boolean; + history_length: number; + internal_purpose: InternalPromptPurpose; }; export function chatResultFeedbackKindToTelemetryValue( diff --git a/src/test/ai-accuracy-tests/ai-accuracy-tests.ts b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts index e5059ce30..1b3d45636 100644 --- a/src/test/ai-accuracy-tests/ai-accuracy-tests.ts +++ b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts @@ -19,7 +19,7 @@ import { } from './create-test-results-html-page'; import { anyOf, runCodeInMessage } from './assertions'; import { Prompts } from '../../participant/prompts'; -import type { PromptResult } from '../../participant/prompts/promptBase'; +import type { ModelInput } from '../../participant/prompts/promptBase'; const numberOfRunsPerTest = 1; @@ -490,7 +490,7 @@ const buildMessages = async ({ }: { testCase: TestCase; fixtures: Fixtures; -}): Promise => { +}): Promise => { switch (testCase.type) { case 'intent': return Prompts.intent.buildMessages({ diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 5a7e69cc8..d11cf0b24 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -11,7 +11,10 @@ import ConnectionController from '../../../connectionController'; import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; import { ExtensionContextStub } from '../stubs'; -import type { ParticipantPromptProperties } from '../../../telemetry/telemetryService'; +import type { + InternalPromptPurpose, + ParticipantPromptProperties, +} from '../../../telemetry/telemetryService'; import TelemetryService, { TelemetryEventTypes, } from '../../../telemetry/telemetryService'; @@ -81,18 +84,28 @@ suite('Participant Controller Test Suite', function () { const assertCommandTelemetry = ( command: string, chatRequest: vscode.ChatRequest, - { expectSampleDocs = false } + { + expectSampleDocs = false, + callIndex = 0, + expectedCallCount, + expectedInternalPurpose = undefined, + }: { + expectSampleDocs?: boolean; + callIndex: number; + expectedCallCount: number; + expectedInternalPurpose?: InternalPromptPurpose; + } ): void => { - expect(telemetryTrackStub).to.have.called; - expect(telemetryTrackStub.lastCall.args[0]).to.equal( - 'Participant Prompt Submitted' - ); + expect(telemetryTrackStub.callCount).to.equal(expectedCallCount); - const properties = telemetryTrackStub.lastCall - .args[1] as ParticipantPromptProperties; + const call = telemetryTrackStub.getCalls()[callIndex]; + expect(call.args[0]).to.equal('Participant Prompt Submitted'); + + const properties = call.args[1] as ParticipantPromptProperties; expect(properties.command).to.equal(command); expect(properties.has_sample_documents).to.equal(expectSampleDocs); + expect(properties.history_length).to.equal(chatContextStub.history.length); // Total message length includes participant as well as user prompt expect(properties.total_message_length).to.be.greaterThan( @@ -104,6 +117,7 @@ suite('Participant Controller Test Suite', function () { expect(properties.user_input_length).to.be.greaterThanOrEqual( chatRequest.prompt.length ); + expect(properties.internal_purpose).to.equal(expectedInternalPurpose); }; beforeEach(function () { @@ -418,7 +432,9 @@ suite('Participant Controller Test Suite', function () { ); expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; assertCommandTelemetry('query', chatRequestMock, { - expectSampleDocs: false, + callIndex: 1, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', }); }); }); @@ -533,7 +549,14 @@ suite('Participant Controller Test Suite', function () { }); assertCommandTelemetry('generic', chatRequestMock, { - expectSampleDocs: false, + expectedCallCount: 2, + callIndex: 0, + expectedInternalPurpose: 'intent', + }); + + assertCommandTelemetry('generic', chatRequestMock, { + expectedCallCount: 2, + callIndex: 1, }); }); }); @@ -565,7 +588,14 @@ suite('Participant Controller Test Suite', function () { }); assertCommandTelemetry('query', chatRequestMock, { - expectSampleDocs: false, + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + expectedCallCount: 2, }); }); @@ -594,7 +624,14 @@ suite('Participant Controller Test Suite', function () { ); assertCommandTelemetry('query', chatRequestMock, { - expectSampleDocs: false, + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + expectedCallCount: 2, }); }); @@ -663,8 +700,16 @@ suite('Participant Controller Test Suite', function () { ']\n' ); + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, + callIndex: 1, + expectedCallCount: 2, }); }); @@ -711,8 +756,16 @@ suite('Participant Controller Test Suite', function () { '}\n' ); + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, + callIndex: 1, + expectedCallCount: 2, }); }); @@ -757,8 +810,16 @@ suite('Participant Controller Test Suite', function () { '}\n' ); + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, + callIndex: 1, + expectedCallCount: 2, }); }); @@ -799,7 +860,14 @@ suite('Participant Controller Test Suite', function () { expect(messages[1].content).to.not.include('Sample documents'); assertCommandTelemetry('query', chatRequestMock, { - expectSampleDocs: false, + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + expectedCallCount: 2, }); }); }); @@ -816,7 +884,14 @@ suite('Participant Controller Test Suite', function () { expect(messages[1].content).to.not.include('Sample documents'); assertCommandTelemetry('query', chatRequestMock, { - expectSampleDocs: false, + callIndex: 0, + expectedCallCount: 2, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + expectedCallCount: 2, }); }); }); From 520bfb584d2d68dbd2173e0bb909b030f065bcb2 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Fri, 27 Sep 2024 22:10:05 +0200 Subject: [PATCH 6/6] history_length -> history_size --- src/participant/prompts/promptBase.ts | 2 +- src/telemetry/telemetryService.ts | 2 +- src/test/suite/participant/participant.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index 2885be48d..949b4f3d0 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -98,7 +98,7 @@ export abstract class PromptBase { user_input_length: request.prompt.length, has_sample_documents: hasSampleDocs, command: request.command || 'generic', - history_length: context.history.length, + history_size: context.history.length, internal_purpose: this.internalPurposeForTelemetry, }; } diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index bfbc200d6..930f7e950 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -115,7 +115,7 @@ export type ParticipantPromptProperties = { user_input_length: number; total_message_length: number; has_sample_documents: boolean; - history_length: number; + history_size: number; internal_purpose: InternalPromptPurpose; }; diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index d11cf0b24..4736b4d65 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -105,7 +105,7 @@ suite('Participant Controller Test Suite', function () { expect(properties.command).to.equal(command); expect(properties.has_sample_documents).to.equal(expectSampleDocs); - expect(properties.history_length).to.equal(chatContextStub.history.length); + expect(properties.history_size).to.equal(chatContextStub.history.length); // Total message length includes participant as well as user prompt expect(properties.total_message_length).to.be.greaterThan(