diff --git a/src/participant/participant.ts b/src/participant/participant.ts index a613aadc4..cbc574b91 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -190,14 +190,21 @@ export default class ParticipantController { modelInput: ModelInput; stream: vscode.ChatResponseStream; token: vscode.CancellationToken; - }): Promise { + }): Promise<{ outputLength: number }> { const chatResponse = await this._getChatResponse({ modelInput, token, }); + + let length = 0; for await (const fragment of chatResponse.text) { stream.markdown(fragment); + length += fragment.length; } + + return { + outputLength: length, + }; } _streamCodeBlockActions({ @@ -236,22 +243,34 @@ export default class ParticipantController { modelInput: ModelInput; stream: vscode.ChatResponseStream; token: vscode.CancellationToken; - }): Promise { + }): Promise<{ + outputLength: number; + hasCodeBlock: boolean; + }> { const chatResponse = await this._getChatResponse({ modelInput, token, }); + let outputLength = 0; + let hasCodeBlock = false; await processStreamWithIdentifiers({ processStreamFragment: (fragment: string) => { stream.markdown(fragment); + outputLength += fragment.length; }, onStreamIdentifier: (content: string) => { this._streamCodeBlockActions({ runnableContent: content, stream }); + hasCodeBlock = true; }, inputIterable: chatResponse.text, identifier: codeBlockIdentifier, }); + + return { + outputLength, + hasCodeBlock, + }; } // This will stream all of the response content and create a string from it. @@ -287,10 +306,19 @@ export default class ParticipantController { connectionNames: this._getConnectionNames(), }); - await this.streamChatResponseContentWithCodeActions({ - modelInput, - token, - stream, + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + token, + stream, + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'generic', + has_cta: false, + found_namespace: false, + has_runnable_content: hasCodeBlock, + output_length: outputLength, }); return genericRequestChatResult(context.history); @@ -995,6 +1023,7 @@ export default class ParticipantController { context, token, }); + if (!databaseName || !collectionName) { return await this._askForNamespace({ command: '/schema', @@ -1056,7 +1085,7 @@ export default class ParticipantController { connectionNames: this._getConnectionNames(), ...(sampleDocuments ? { sampleDocuments } : {}), }); - await this.streamChatResponse({ + const response = await this.streamChatResponse({ modelInput, stream, token, @@ -1072,6 +1101,14 @@ export default class ParticipantController { ], }); + this._telemetryService.trackCopilotParticipantResponse({ + command: 'schema', + has_cta: true, + found_namespace: true, + has_runnable_content: false, + output_length: response.outputLength, + }); + return schemaRequestChatResult(context.history); } @@ -1160,10 +1197,19 @@ export default class ParticipantController { ...(sampleDocuments ? { sampleDocuments } : {}), }); - await this.streamChatResponseContentWithCodeActions({ - modelInput, - stream, - token, + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + stream, + token, + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'query', + has_cta: false, + found_namespace: true, + has_runnable_content: hasCodeBlock, + output_length: outputLength, }); return queryRequestChatResult(context.history); @@ -1239,11 +1285,12 @@ export default class ParticipantController { connectionNames: this._getConnectionNames(), }); - await this.streamChatResponseContentWithCodeActions({ - modelInput, - stream, - token, - }); + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + stream, + token, + }); this._streamResponseReference({ reference: { @@ -1252,6 +1299,14 @@ export default class ParticipantController { }, stream, }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'docs/copilot', + has_cta: true, + found_namespace: false, + has_runnable_content: hasCodeBlock, + output_length: outputLength, + }); } _streamResponseReference({ @@ -1307,6 +1362,14 @@ export default class ParticipantController { if (docsResult.responseContent) { stream.markdown(docsResult.responseContent); } + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'docs/chatbot', + has_cta: !!docsResult.responseReferences, + found_namespace: false, + has_runnable_content: false, + output_length: docsResult.responseContent?.length ?? 0, + }); } catch (error) { // If the docs chatbot API is not available, fall back to Copilot’s LLM and include // the MongoDB documentation link for users to go to our documentation site directly. diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 930f7e950..93220661e 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -119,6 +119,14 @@ export type ParticipantPromptProperties = { internal_purpose: InternalPromptPurpose; }; +export type ParticipantResponseProperties = { + command: string; + has_cta: boolean; + has_runnable_content: boolean; + found_namespace: boolean; + output_length: number; +}; + export function chatResultFeedbackKindToTelemetryValue( kind: vscode.ChatResultFeedbackKind ): TelemetryFeedbackKind { @@ -148,7 +156,9 @@ type TelemetryEventProperties = | SavedConnectionsLoadedProperties | SurveyActionProperties | ParticipantFeedbackProperties - | ParticipantResponseFailedProperties; + | ParticipantResponseFailedProperties + | ParticipantPromptProperties + | ParticipantResponseProperties; export enum TelemetryEventTypes { PLAYGROUND_CODE_EXECUTED = 'Playground Code Executed', @@ -172,6 +182,7 @@ export enum TelemetryEventTypes { PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', + PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', } export enum ParticipantErrorTypes { @@ -438,4 +449,8 @@ export default class TelemetryService { trackCopilotParticipantPrompt(stats: ParticipantPromptProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats); } + + trackCopilotParticipantResponse(props: ParticipantResponseProperties): void { + this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props); + } } diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index adde458d8..b46f19294 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -14,6 +14,7 @@ import { ExtensionContextStub } from '../stubs'; import type { InternalPromptPurpose, ParticipantPromptProperties, + ParticipantResponseProperties, } from '../../../telemetry/telemetryService'; import TelemetryService, { TelemetryEventTypes, @@ -87,16 +88,14 @@ suite('Participant Controller Test Suite', function () { { expectSampleDocs = false, callIndex = 0, - expectedCallCount, expectedInternalPurpose = undefined, }: { expectSampleDocs?: boolean; callIndex: number; - expectedCallCount: number; expectedInternalPurpose?: InternalPromptPurpose; } ): void => { - expect(telemetryTrackStub.callCount).to.equal(expectedCallCount); + expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); const call = telemetryTrackStub.getCalls()[callIndex]; expect(call.args[0]).to.equal('Participant Prompt Submitted'); @@ -120,6 +119,33 @@ suite('Participant Controller Test Suite', function () { expect(properties.internal_purpose).to.equal(expectedInternalPurpose); }; + const assertResponseTelemetry = ( + command: string, + { + callIndex = 0, + hasCTA = false, + hasRunnableContent = false, + foundNamespace = false, + }: { + callIndex: number; + hasCTA?: boolean; + hasRunnableContent?: boolean; + foundNamespace?: boolean; + } + ): void => { + expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); + const call = telemetryTrackStub.getCalls()[callIndex]; + expect(call.args[0]).to.equal('Participant Response Generated'); + + const properties = call.args[1] as ParticipantResponseProperties; + + expect(properties.command).to.equal(command); + expect(properties.found_namespace).to.equal(foundNamespace); + expect(properties.has_cta).to.equal(hasCTA); + expect(properties.has_runnable_content).to.equal(hasRunnableContent); + expect(properties.output_length).to.be.greaterThan(0); + }; + beforeEach(function () { testStorageController = new StorageController(extensionContextStub); testStatusView = new StatusView(extensionContextStub); @@ -433,7 +459,6 @@ suite('Participant Controller Test Suite', function () { expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); }); @@ -549,15 +574,18 @@ suite('Participant Controller Test Suite', function () { }); assertCommandTelemetry('generic', chatRequestMock, { - expectedCallCount: 2, callIndex: 0, expectedInternalPurpose: 'intent', }); assertCommandTelemetry('generic', chatRequestMock, { - expectedCallCount: 2, callIndex: 1, }); + + assertResponseTelemetry('generic', { + callIndex: 2, + hasRunnableContent: true, + }); }); }); @@ -589,13 +617,17 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); @@ -625,13 +657,17 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); @@ -702,14 +738,18 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); @@ -758,14 +798,18 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); @@ -812,14 +856,18 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { expectSampleDocs: true, callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); @@ -861,13 +909,17 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); }); @@ -885,13 +937,17 @@ suite('Participant Controller Test Suite', function () { assertCommandTelemetry('query', chatRequestMock, { callIndex: 0, - expectedCallCount: 2, expectedInternalPurpose: 'namespace', }); assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, - expectedCallCount: 2, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, }); }); }); @@ -1341,6 +1397,21 @@ suite('Participant Controller Test Suite', function () { }, ], }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('schema', { + callIndex: 2, + hasCTA: true, + foundNamespace: true, + }); }); test("includes the collection's schema in the request", async function () { @@ -1384,6 +1455,21 @@ Schema: "field", "arrayField" ],`); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('schema', { + callIndex: 2, + hasCTA: true, + foundNamespace: true, + }); }); test('prints a message when no documents are found', async function () { @@ -1397,6 +1483,11 @@ Schema: expect(chatStreamStub?.markdown.getCall(0).args[0]).to.include( 'Unable to generate a schema from the collection, no documents found.' ); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); }); }); }); @@ -1423,7 +1514,8 @@ Schema: json: () => Promise.resolve({ _id: '650b4b260f975ef031016c8a', - messages: [], + content: + 'To connect to MongoDB using mongosh, you can follow these steps', }), }); global.fetch = fetchStub; @@ -1435,6 +1527,10 @@ Schema: await invokeChatHandler(chatRequestMock); expect(fetchStub).to.have.been.called; expect(sendRequestStub).to.have.not.been.called; + + assertResponseTelemetry('docs/chatbot', { + callIndex: 0, + }); }); test('falls back to the copilot model when docs chatbot result is not available', async function () { @@ -1454,7 +1550,9 @@ Schema: expect(sendRequestStub).to.have.been.called; // Expect the error to be reported through the telemetry service - expect(telemetryTrackStub).to.have.been.calledTwice; + expect( + telemetryTrackStub.getCalls() + ).to.have.length.greaterThanOrEqual(2); expect(telemetryTrackStub.firstCall.args[0]).to.equal( TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED ); @@ -1462,6 +1560,11 @@ Schema: const properties = telemetryTrackStub.firstCall.args[1]; expect(properties.command).to.equal('docs'); expect(properties.error_name).to.equal('Docs Chatbot API Issue'); + + assertResponseTelemetry('docs/copilot', { + callIndex: 2, + hasCTA: true, + }); }); }); });