diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 4098014e9..7a0669da3 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -1120,6 +1120,14 @@ export default class ParticipantController { // If the docs chatbot API is not available, fall back to Copilot’s LLM and include // the MongoDB documentation link for users to go to our documentation site directly. log.error(error); + this._telemetryService.track( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED, + { + command: 'docs', + error_name: ParticipantErrorTypes.DOCS_CHATBOT_API, + } + ); + docsResult = await this._handleDocsRequestWithCopilot(...args); } diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index a9608c8d7..53dd2cba5 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -105,7 +105,7 @@ type ParticipantFeedbackProperties = { type ParticipantResponseFailedProperties = { command: string; error_code?: string; - error_name: string; + error_name: ParticipantErrorTypes; }; export function chatResultFeedbackKindToTelemetryValue( @@ -167,6 +167,7 @@ export enum ParticipantErrorTypes { INVALID_PROMPT = 'Invalid Prompt', FILTERED = 'Filtered by Responsible AI Service', OTHER = 'Other', + DOCS_CHATBOT_API = 'Docs Chatbot API Issue', } /** diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 53172b106..91622dac7 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -64,7 +64,7 @@ suite('Participant Controller Test Suite', function () { }; let chatTokenStub; let countTokensStub; - let sendRequestStub; + let sendRequestStub: sinon.SinonStub; let telemetryTrackStub: SinonSpy; const invokeChatHandler = async ( @@ -399,13 +399,6 @@ suite('Participant Controller Test Suite', function () { TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN ); expect(telemetryTrackStub.lastCall.args[1]).to.be.undefined; - - telemetryTrackStub - .getCalls() - .map((call) => call.args[0]) - .filter( - (arg) => arg === TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN - ).length; }); }); @@ -1226,7 +1219,7 @@ Schema: suite('docs command', function () { const initialFetch = global.fetch; - let fetchStub; + let fetchStub: sinon.SinonStub; beforeEach(function () { sendRequestStub.onCall(0).resolves({ @@ -1275,6 +1268,16 @@ Schema: }; await invokeChatHandler(chatRequestMock); expect(sendRequestStub).to.have.been.called; + + // Expect the error to be reported through the telemetry service + sinon.assert.calledOnce(telemetryTrackStub); + expect(telemetryTrackStub.lastCall.args[0]).to.equal( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.command).to.equal('docs'); + expect(properties.error_name).to.equal('Docs Chatbot API Issue'); }); }); });