diff --git a/packages/vscode-extension/src/officeChat/commands/create/helper.ts b/packages/vscode-extension/src/officeChat/commands/create/helper.ts index a6f4a40120..6567d571f4 100644 --- a/packages/vscode-extension/src/officeChat/commands/create/helper.ts +++ b/packages/vscode-extension/src/officeChat/commands/create/helper.ts @@ -25,6 +25,7 @@ import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../ import { OfficeXMLAddinGenerator } from "./officeXMLAddinGenerator/generator"; import { CreateProjectInputs } from "@microsoft/teamsfx-api"; import { core } from "../../../globalVariables"; +import { ProjectMiniData } from "../../types"; export async function matchOfficeProject( request: ChatRequest, @@ -39,8 +40,8 @@ export async function matchOfficeProject( getOfficeProjectMatchSystemPrompt(allOfficeProjectMetadata), new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt), ]; - const t0 = performance.now(); let response = ""; + telemetryData.chatMessages.push(...messages); try { response = await getCopilotResponseAsString("copilot-gpt-4", messages, token); } catch (error) { @@ -50,12 +51,7 @@ export async function matchOfficeProject( telemetryData.markComplete("unsupportedPrompt"); } } - const t1 = performance.now(); - telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(response, t0, t1) - ); - telemetryData.chatMessages.push( - ...messages, + telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, response) ); let matchedProjectId: string; @@ -112,7 +108,7 @@ export function getOfficeTemplateMetadata(): ProjectMetadata[] { export async function showOfficeSampleFileTree( projectMetadata: ProjectMetadata, response: ChatResponseStream -): Promise { +): Promise { response.markdown( "\nWe've found a sample project that matches your description. Take a look at it below." ); @@ -128,7 +124,11 @@ export async function showOfficeSampleFileTree( 20 ); response.filetree(nodes, Uri.file(path.join(tempFolder, downloadUrlInfo.dir))); - return { path: path.join(tempFolder, downloadUrlInfo.dir), host: host }; + const result: ProjectMiniData = { + path: path.join(tempFolder, downloadUrlInfo.dir), + host: host, + }; + return result; } export async function showOfficeTemplateFileTree( diff --git a/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts b/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts index 2c6a08e704..4e07c5cf89 100644 --- a/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts +++ b/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts @@ -13,7 +13,7 @@ import { import { OfficeChatCommand, officeChatParticipantId } from "../../consts"; import { verbatimCopilotInteraction } from "../../../chat/utils"; import { isInputHarmful } from "../../utils"; -import { ICopilotChatOfficeResult } from "../../types"; +import { ICopilotChatOfficeResult, ProjectMiniData } from "../../types"; import { describeOfficeProjectSystemPrompt } from "../../officePrompts"; import { TelemetryEvent } from "../../../telemetry/extTelemetryEvents"; import { ExtTelemetry } from "../../../telemetry/extTelemetry"; @@ -22,6 +22,7 @@ import { localize } from "../../../utils/localizeUtils"; import { Planner } from "../../common/planner"; import { CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID } from "../../consts"; import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../../telemetry"; +import { ProjectMetadata } from "../../../chat/commands/create/types"; export default async function officeCreateCommandHandler( request: ChatRequest, @@ -78,15 +79,17 @@ export default async function officeCreateCommandHandler( ); if (matchedResult.type === "sample") { - const sampleInfos = await showOfficeSampleFileTree(matchedResult, response); - const folder = (sampleInfos as any)?.["path"]; - const hostType = (sampleInfos as any)?.["host"].toLowerCase(); + const sampleInfos: ProjectMiniData = await showOfficeSampleFileTree( + matchedResult, + response + ); + const folder = sampleInfos.path; + const hostType = sampleInfos.host.toLowerCase(); const sampleTitle = localize("teamstoolkit.chatParticipants.create.sample"); officeChatTelemetryData.setHostType(hostType); - const matchResultInfo = "sample"; response.button({ command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID, - arguments: [folder, officeChatTelemetryData.requestId, matchResultInfo], + arguments: [folder, officeChatTelemetryData.requestId, matchedResult.type], title: sampleTitle, }); } else { @@ -94,10 +97,9 @@ export default async function officeCreateCommandHandler( const tmpFolder = await showOfficeTemplateFileTree(matchedResult.data as any, response); const templateTitle = localize("teamstoolkit.chatParticipants.create.template"); officeChatTelemetryData.setHostType(tmpHostType); - const tmpmatchResultInfo = "template"; response.button({ command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID, - arguments: [tmpFolder, officeChatTelemetryData.requestId, tmpmatchResultInfo], + arguments: [tmpFolder, officeChatTelemetryData.requestId, matchedResult.type], title: templateTitle, }); } diff --git a/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts b/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts index 9085f97765..cd93a7199c 100644 --- a/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts +++ b/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts @@ -75,13 +75,8 @@ export default async function officeNextStepCommandHandler( if (s.description instanceof Function) { s.description = s.description(status); } - const t0 = performance.now(); const stepDescription = await describeOfficeStep(s, token, officeChatTelemetryData); - const t1 = performance.now(); - officeChatTelemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(stepDescription, t0, t1) - ); - officeChatTelemetryData.chatMessages.push( + officeChatTelemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, stepDescription) ); const title = s.docLink ? `[${s.title}](${s.docLink})` : s.title; diff --git a/packages/vscode-extension/src/officeChat/common/planner.ts b/packages/vscode-extension/src/officeChat/common/planner.ts index d3a1ac0760..88682f0529 100644 --- a/packages/vscode-extension/src/officeChat/common/planner.ts +++ b/packages/vscode-extension/src/officeChat/common/planner.ts @@ -86,8 +86,7 @@ export class Planner { ${localize("teamstoolkit.chatParticipants.officeAddIn.printer.outputTemplate.intro")}\n ${purified} `); - const spec = new Spec(purified); - spec.appendix.telemetryData.requestId = telemetryData.requestId; + const spec = new Spec(purified, telemetryData.requestId); try { for (let index = 0; index < candidates.length; index++) { const candidate = candidates[index]; @@ -147,14 +146,11 @@ ${purified} telemetryData.setHostType(spec.appendix.host?.toLowerCase()); telemetryData.setTimeToFirstToken(spec.appendix.telemetryData.timeToFirstToken); telemetryData.setRelatedSampleName(spec.appendix.telemetryData.relatedSampleName.toString()); - // telemetryData.setCodeClassAndMembers( - // spec.appendix.telemetryData.codeClassAndMembers.toString() - // ); for (const chatMessage of spec.appendix.telemetryData.chatMessages) { telemetryData.chatMessages.push(chatMessage); } - for (const responseTokensPerRequest of spec.appendix.telemetryData.responseTokensPerRequest) { - telemetryData.responseTokensPerRequest.push(responseTokensPerRequest); + for (const responseChatMessage of spec.appendix.telemetryData.responseChatMessages) { + telemetryData.responseChatMessages.push(responseChatMessage); } const debugInfo = ` ## Time cost:\n diff --git a/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts b/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts index 015c612911..e0eae476bc 100644 --- a/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts +++ b/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts @@ -104,14 +104,9 @@ export class SampleProvider { } countOfLLMInvoke += 1; - const timeStart = performance.now(); const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token); - const timeEnd = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, timeStart, timeEnd) - ); - spec.appendix.telemetryData.chatMessages.push( - sampleMessage, + spec.appendix.telemetryData.chatMessages.push(sampleMessage); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); const returnObject: { picked: string[] } = JSON.parse( @@ -301,14 +296,9 @@ export class SampleProvider { LanguageModelChatMessageRole.User, getMoreRelevantMethodsOrPropertiesPrompt ); - const t0 = performance.now(); const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - sampleMessage, + spec.appendix.telemetryData.chatMessages.push(sampleMessage); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); let returnObject: { picked: string[] } = { picked: [] }; diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts b/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts index 8b1eed6508..b70a18ff5b 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts @@ -57,18 +57,13 @@ Let's think it step by step. new LanguageModelChatMessage(LanguageModelChatMessageRole.User, systemPrompt), new LanguageModelChatMessage(LanguageModelChatMessageRole.User, userPrompt), ]; - const t0 = performance.now(); const copilotResponse = await getCopilotResponseAsString( "copilot-gpt-3.5-turbo", messages, token ); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts b/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts index e3592560f8..b25b5b7fe6 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts @@ -212,18 +212,13 @@ ${spec.appendix.codeExplanation new LanguageModelChatMessage(LanguageModelChatMessageRole.User, userPrompt), new LanguageModelChatMessage(LanguageModelChatMessageRole.System, defaultSystemPrompt), ]; - const t0 = performance.now(); let copilotResponse = await getCopilotResponseAsString( "copilot-gpt-3.5-turbo", // "copilot-gpt-4", // "copilot-gpt-3.5-turbo", messages, token ); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); let copilotRet: { @@ -329,18 +324,13 @@ ${spec.appendix.codeExplanation ); } - const t0 = performance.now(); let copilotResponse = await getCopilotResponseAsString( "copilot-gpt-4", //"copilot-gpt-4", // "copilot-gpt-3.5-turbo", messages, token ); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); let copilotRet: { @@ -500,14 +490,9 @@ ${spec.appendix.codeExplanation } console.debug(`token count: ${msgCount}, number of messages remains: ${messages.length}.`); - const t0 = performance.now(); const copilotResponse = await getCopilotResponseAsString(model, messages, token); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); // extract the code snippet and the api list out diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts b/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts index 70827dbc18..6cbc6abe4e 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts @@ -353,14 +353,9 @@ export class CodeIssueCorrector implements ISkill { msgCount = countMessagesTokens(messages); } // console.debug(`token count: ${msgCount}, number of messages remains: ${messages.length}.`); - const t0 = performance.now(); const copilotResponse = await getCopilotResponseAsString(model, messages, token); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse) ); // extract the code snippet diff --git a/packages/vscode-extension/src/officeChat/common/skills/spec.ts b/packages/vscode-extension/src/officeChat/common/skills/spec.ts index eaab6860a7..23206926dc 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/spec.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/spec.ts @@ -23,10 +23,9 @@ export class Spec { requestId: string; isHarmful: boolean; relatedSampleName: string[]; - codeClassAndMembers: string[]; timeToFirstToken: DOMHighResTimeStamp; chatMessages: LanguageModelChatMessage[]; - responseTokensPerRequest: number[]; + responseChatMessages: LanguageModelChatMessage[]; properties: { [key: string]: string }; measurements: { [key: string]: number }; }; @@ -34,7 +33,7 @@ export class Spec { shouldContinue: boolean; }; - constructor(userInput: string) { + constructor(userInput: string, requestId?: string) { this.userInput = userInput; this.taskSummary = ""; this.sections = []; @@ -49,13 +48,12 @@ export class Spec { apiDeclarationsReference: new Map(), isCustomFunction: false, telemetryData: { - requestId: "", + requestId: requestId ? requestId : "", isHarmful: false, relatedSampleName: [], - codeClassAndMembers: [], timeToFirstToken: 0, chatMessages: [], - responseTokensPerRequest: [], + responseChatMessages: [], properties: {}, measurements: {}, }, diff --git a/packages/vscode-extension/src/officeChat/telemetry.ts b/packages/vscode-extension/src/officeChat/telemetry.ts index f2abb97e93..6e6491d46e 100644 --- a/packages/vscode-extension/src/officeChat/telemetry.ts +++ b/packages/vscode-extension/src/officeChat/telemetry.ts @@ -2,8 +2,7 @@ // Licensed under the MIT license. import { LanguageModelChatMessage, LanguageModelChatMessageRole } from "vscode"; -import { ChatTelemetryData } from "../chat/telemetry"; -import { ITelemetryData } from "../chat/types"; +import { IChatTelemetryData, ITelemetryData } from "../chat/types"; import { Correlator, getUuid } from "@microsoft/teamsfx-core"; import { countMessagesTokens } from "../chat/utils"; import { @@ -17,19 +16,18 @@ export enum OfficeChatTelemetryBlockReasonEnum { OffTopic = "Off Topic", LanguageModelError = "LanguageModel Error", } -export class OfficeChatTelemetryData extends ChatTelemetryData { +export class OfficeChatTelemetryData implements IChatTelemetryData { public static requestData: { [key: string]: OfficeChatTelemetryData } = {}; telemetryData: ITelemetryData; chatMessages: LanguageModelChatMessage[] = []; + responseChatMessages: LanguageModelChatMessage[] = []; command: string; requestId: string; startTime: number; hostType: string; relatedSampleName: string; - codeClassAndMembers: string; timeToFirstToken: number; - responseTokensPerRequest: number[]; blockReason?: string; // participant name participantId: string; @@ -45,15 +43,12 @@ export class OfficeChatTelemetryData extends ChatTelemetryData { } constructor(command: string, requestId: string, startTime: number, participantId: string) { - super(command, requestId, startTime, participantId); this.command = command; this.requestId = requestId; this.startTime = startTime; this.participantId = participantId; this.hostType = ""; this.relatedSampleName = ""; - this.codeClassAndMembers = ""; - this.responseTokensPerRequest = []; this.timeToFirstToken = -1; const telemetryData: ITelemetryData = { properties: {}, measurements: {} }; @@ -79,17 +74,6 @@ export class OfficeChatTelemetryData extends ChatTelemetryData { return OfficeChatTelemetryData.requestData[requestId]; } - static calculateResponseTokensPerRequest( - response: string, - t0: DOMHighResTimeStamp, - t1: DOMHighResTimeStamp - ) { - const responseTokens = countMessagesTokens([ - new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, response), - ]); - return responseTokens / ((t1 - t0) / 1000); - } - setHostType(hostType: string) { this.hostType = hostType; } @@ -98,15 +82,11 @@ export class OfficeChatTelemetryData extends ChatTelemetryData { this.relatedSampleName = relatedSampleName; } - setCodeClassAndMembers(codeClassAndMembers: string) { - this.codeClassAndMembers = codeClassAndMembers; - } - setTimeToFirstToken(t0?: DOMHighResTimeStamp) { if (t0) { - this.timeToFirstToken = t0 - this.startTime; + this.timeToFirstToken = (t0 - this.startTime) / 1000; } else { - this.timeToFirstToken = performance.now() - this.startTime; + this.timeToFirstToken = (performance.now() - this.startTime) / 1000; } } @@ -118,6 +98,10 @@ export class OfficeChatTelemetryData extends ChatTelemetryData { return countMessagesTokens(this.chatMessages); } + responseChatMessagesTokenCount(): number { + return countMessagesTokens(this.responseChatMessages); + } + extendBy(properties?: { [key: string]: string }, measurements?: { [key: string]: number }) { this.telemetryData.properties = { ...this.telemetryData.properties, ...properties }; this.telemetryData.measurements = { ...this.telemetryData.measurements, ...measurements }; @@ -127,25 +111,26 @@ export class OfficeChatTelemetryData extends ChatTelemetryData { if (!this.hasComplete) { this.telemetryData.properties[TelemetryProperty.Success] = TelemetrySuccess.Yes; this.telemetryData.properties[TelemetryProperty.CopilotChatCompleteType] = completeType; - if (this.blockReason) { + if (this.blockReason && this.blockReason !== "") { this.telemetryData.properties[TelemetryProperty.CopilotChatBlockReason] = this.blockReason; } this.telemetryData.properties[TelemetryProperty.HostType] = this.hostType; this.telemetryData.properties[TelemetryProperty.CopilotChatRelatedSampleName] = this.relatedSampleName; - // this.telemetryData.properties[TelemetryProperty.CopilotChatCodeClassAndMembers] = - // this.codeClassAndMembers; - this.telemetryData.properties[TelemetryProperty.CopilotChatResponseTokensPerRequest] = - this.responseTokensPerRequest.toString(); this.telemetryData.measurements[TelemetryProperty.CopilotChatTimeToFirstToken] = this.timeToFirstToken; this.telemetryData.measurements[TelemetryProperty.CopilotChatTimeToComplete] = - performance.now() - this.startTime; - this.telemetryData.measurements[TelemetryProperty.CopilotChatTokenCount] = + (performance.now() - this.startTime) / 1000; + this.telemetryData.measurements[TelemetryProperty.CopilotRequestChatCount] = this.chatMessagesTokenCount(); - this.telemetryData.measurements[TelemetryProperty.CopilotChatTotalTokensPerSecond] = - this.telemetryData.measurements[TelemetryProperty.CopilotChatTokenCount] / - (this.telemetryData.measurements[TelemetryProperty.CopilotChatTimeToComplete] / 1000); + this.telemetryData.measurements[TelemetryProperty.CopilotResponseChatCount] = + this.responseChatMessagesTokenCount(); + this.telemetryData.measurements[TelemetryProperty.CopilotRequestChatCountPerSecond] = + this.telemetryData.measurements[TelemetryProperty.CopilotRequestChatCount] / + this.telemetryData.measurements[TelemetryProperty.CopilotChatTimeToComplete]; + this.telemetryData.measurements[TelemetryProperty.CopilotResponseChatCountPerSecond] = + this.telemetryData.measurements[TelemetryProperty.CopilotResponseChatCount] / + this.telemetryData.measurements[TelemetryProperty.CopilotChatTimeToComplete]; this.hasComplete = true; } } diff --git a/packages/vscode-extension/src/officeChat/types.ts b/packages/vscode-extension/src/officeChat/types.ts index def5500626..8b48f3cf6f 100644 --- a/packages/vscode-extension/src/officeChat/types.ts +++ b/packages/vscode-extension/src/officeChat/types.ts @@ -11,3 +11,8 @@ export interface ICopilotChatOfficeResultMetadata { export interface ICopilotChatOfficeResult extends ChatResult { readonly metadata?: ICopilotChatOfficeResultMetadata; } + +export type ProjectMiniData = { + path: string; + host: string; +}; diff --git a/packages/vscode-extension/src/officeChat/utils.ts b/packages/vscode-extension/src/officeChat/utils.ts index 8283f2cf6d..03a260573a 100644 --- a/packages/vscode-extension/src/officeChat/utils.ts +++ b/packages/vscode-extension/src/officeChat/utils.ts @@ -33,18 +33,13 @@ export async function purifyUserMessage( new LanguageModelChatMessage(LanguageModelChatMessageRole.User, userMessagePrompt), new LanguageModelChatMessage(LanguageModelChatMessageRole.System, systemPrompt), ]; - const t0 = performance.now(); const purifiedResult = await getCopilotResponseAsString( "copilot-gpt-4", purifyUserMessage, token ); - const t1 = performance.now(); - telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(purifiedResult, t0, t1) - ); - telemetryData.chatMessages.push( - ...purifyUserMessage, + telemetryData.chatMessages.push(...purifyUserMessage); + telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, purifiedResult) ); if ( @@ -63,14 +58,9 @@ export async function isInputHarmful( telemetryData: OfficeChatTelemetryData ): Promise { const messages = buildDynamicPrompt(inputRai, request.prompt).messages; - const t0 = performance.now(); let response = await getCopilotResponseAsString("copilot-gpt-4", messages, token); - const t1 = performance.now(); - telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(response, t0, t1) - ); - telemetryData.chatMessages.push( - ...messages, + telemetryData.chatMessages.push(...messages); + telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, response) ); if (!response) { @@ -105,14 +95,9 @@ async function isContentHarmful( spec: Spec ): Promise { async function getIsHarmfulResponseAsync() { - const t0 = performance.now(); const isHarmfulResponse = await getCopilotResponseAsString("copilot-gpt-4", messages, token); - const t1 = performance.now(); - spec.appendix.telemetryData.responseTokensPerRequest.push( - OfficeChatTelemetryData.calculateResponseTokensPerRequest(isHarmfulResponse, t0, t1) - ); - spec.appendix.telemetryData.chatMessages.push( - ...messages, + spec.appendix.telemetryData.chatMessages.push(...messages); + spec.appendix.telemetryData.responseChatMessages.push( new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, isHarmfulResponse) ); if ( diff --git a/packages/vscode-extension/src/telemetry/extTelemetryEvents.ts b/packages/vscode-extension/src/telemetry/extTelemetryEvents.ts index 302cfb78b4..50a49356e1 100644 --- a/packages/vscode-extension/src/telemetry/extTelemetryEvents.ts +++ b/packages/vscode-extension/src/telemetry/extTelemetryEvents.ts @@ -391,11 +391,11 @@ export enum TelemetryProperty { CopilotMatchResultType = "copilot-match-result-type", CopilotChatBlockReason = "copilot-chat-block-reason", CopilotChatRelatedSampleName = "copilot-chat-related-sample-name", - CopilotChatCodeClassAndMembers = "copilot-chat-code-class-and-members", CopilotChatTimeToFirstToken = "copilot-chat-time-to-first-token", - CopilotChatTotalTokensPerSecond = "copilot-chat-total-tokens-per-second", - CopilotChatResponseTokensPerRequest = "copilot-chat-response-tokens-per-request", - CopilotChatTotalTokens = "copilot-chat-total-tokens", + CopilotRequestChatCountPerSecond = "copilot-chat-total-tokens-per-second", + CopilotResponseChatCountPerSecond = "copilot-chat-total-tokens-per-second", + CopilotRequestChatCount = "copilot-request-chat-total-tokens", + CopilotResponseChatCount = "copilot-response-chat-total-tokens", } export enum TelemetryMeasurements { diff --git a/packages/vscode-extension/test/officeChat/commands/create/helper.test.ts b/packages/vscode-extension/test/officeChat/commands/create/helper.test.ts index 60a6b349f9..3a6285b2b7 100644 --- a/packages/vscode-extension/test/officeChat/commands/create/helper.test.ts +++ b/packages/vscode-extension/test/officeChat/commands/create/helper.test.ts @@ -46,7 +46,7 @@ describe("File: office chat create helper", () => { }; }); officeChatTelemetryDataMock.chatMessages = []; - officeChatTelemetryDataMock.responseTokensPerRequest = []; + officeChatTelemetryDataMock.responseChatMessages = []; sandbox .stub(OfficeChatTelemetryData, "createByParticipant") .returns(officeChatTelemetryDataMock); diff --git a/packages/vscode-extension/test/officeChat/commands/create/officeCreateCommandHandler.test.ts b/packages/vscode-extension/test/officeChat/commands/create/officeCreateCommandHandler.test.ts index 52b93024cc..3f5d99f7e0 100644 --- a/packages/vscode-extension/test/officeChat/commands/create/officeCreateCommandHandler.test.ts +++ b/packages/vscode-extension/test/officeChat/commands/create/officeCreateCommandHandler.test.ts @@ -11,6 +11,7 @@ import { CancellationToken } from "../../../mocks/vsc"; import { ProjectMetadata } from "../../../../src/chat/commands/create/types"; import { Planner } from "../../../../src/officeChat/common/planner"; import { OfficeChatTelemetryData } from "../../../../src/officeChat/telemetry"; +import { ProjectMiniData } from "../../../../src/officeChat/types"; chai.use(chaipromised); @@ -82,7 +83,13 @@ describe("File: officeCreateCommandHandler", () => { } as ProjectMetadata; sandbox.stub(officeChatUtil, "isInputHarmful").resolves(false); sandbox.stub(helper, "matchOfficeProject").resolves(fakedSample); - const showOfficeSampleFileTreeStub = sandbox.stub(helper, "showOfficeSampleFileTree"); + const mockProjectMiniData: ProjectMiniData = { + path: "", + host: "", + }; + const showOfficeSampleFileTreeStub = sandbox + .stub(helper, "showOfficeSampleFileTree") + .resolves(mockProjectMiniData); sandbox.stub(chatUtil, "verbatimCopilotInteraction"); const response = { markdown: sandbox.stub(), diff --git a/packages/vscode-extension/test/officeChat/commands/nextstep/officeNextstepCommandHelper.test.ts b/packages/vscode-extension/test/officeChat/commands/nextstep/officeNextstepCommandHelper.test.ts index 44c7420bad..e5896d6254 100644 --- a/packages/vscode-extension/test/officeChat/commands/nextstep/officeNextstepCommandHelper.test.ts +++ b/packages/vscode-extension/test/officeChat/commands/nextstep/officeNextstepCommandHelper.test.ts @@ -32,7 +32,7 @@ describe("office steps: officeNextStepCommandHandler", () => { return undefined; }); chatTelemetryDataMock.chatMessages = []; - chatTelemetryDataMock.responseTokensPerRequest = []; + chatTelemetryDataMock.responseChatMessages = []; sandbox.stub(OfficeChatTelemetryData, "createByParticipant").returns(chatTelemetryDataMock); sandbox.stub(ExtTelemetry, "sendTelemetryEvent"); }); diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeExplainer.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeExplainer.test.ts index 5d42c37e73..3cd66c8924 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeExplainer.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeExplainer.test.ts @@ -35,13 +35,12 @@ describe("CodeExplainer", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), ], - responseTokensPerRequest: [], + responseChatMessages: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, @@ -105,13 +104,12 @@ describe("CodeExplainer", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), ], - responseTokensPerRequest: [], + responseChatMessages: [], properties: { property1: "value1", property2: "value2", diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts index 558bfc9d4e..9d4d9de144 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts @@ -36,13 +36,15 @@ describe("codeGenerator", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, @@ -106,13 +108,15 @@ describe("codeGenerator", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2", diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts index cc9d506908..6a13674334 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts @@ -39,13 +39,15 @@ describe("CodeIssueCorrector", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, @@ -109,13 +111,15 @@ describe("CodeIssueCorrector", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2", diff --git a/packages/vscode-extension/test/officeChat/common/skills/printer.test.ts b/packages/vscode-extension/test/officeChat/common/skills/printer.test.ts index d65932952a..e7faace39e 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/printer.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/printer.test.ts @@ -35,13 +35,15 @@ describe("printer", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, @@ -105,13 +107,15 @@ describe("printer", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2", diff --git a/packages/vscode-extension/test/officeChat/common/skills/projectCreator.test.ts b/packages/vscode-extension/test/officeChat/common/skills/projectCreator.test.ts index 79bc6a4af6..6f5f86280e 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/projectCreator.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/projectCreator.test.ts @@ -35,13 +35,15 @@ describe("projectCreator", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, @@ -105,13 +107,15 @@ describe("projectCreator", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), ], - responseTokensPerRequest: [], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), + ], properties: { property1: "value1", property2: "value2", diff --git a/packages/vscode-extension/test/officeChat/common/skills/skillset.test.ts b/packages/vscode-extension/test/officeChat/common/skills/skillset.test.ts index 115377e5fa..2720472aa7 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/skillset.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/skillset.test.ts @@ -35,13 +35,15 @@ describe("skillset", () => { requestId: "Id", isHarmful: false, relatedSampleName: ["sample1", "sample2"], - codeClassAndMembers: ["class1", "class2"], timeToFirstToken: 0, chatMessages: [ - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message1"), - new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "message2"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "requestMessage2"), + ], + responseChatMessages: [ + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage1"), + new LanguageModelChatMessage(LanguageModelChatMessageRole.User, "responseMessage2"), ], - responseTokensPerRequest: [], properties: { property1: "value1", property2: "value2" }, measurements: { measurement1: 1, measurement2: 2 }, }, diff --git a/packages/vscode-extension/test/officeChat/telemetry.test.ts b/packages/vscode-extension/test/officeChat/telemetry.test.ts new file mode 100644 index 0000000000..6e7d6989cf --- /dev/null +++ b/packages/vscode-extension/test/officeChat/telemetry.test.ts @@ -0,0 +1,253 @@ +import sinon from "ts-sinon"; +import * as chai from "chai"; +import { OfficeChatTelemetryData } from "../../src/officeChat/telemetry"; +import { Correlator } from "@microsoft/teamsfx-core"; +import { + TelemetryProperty, + TelemetrySuccess, + TelemetryTriggerFrom, +} from "../../src/telemetry/extTelemetryEvents"; +import * as utils from "../../src/chat/utils"; +import * as coreTools from "@microsoft/teamsfx-core/build/common/stringUtils"; + +describe("OfficeChatTelemetryData", () => { + const sandbox = sinon.createSandbox(); + + afterEach(() => { + sandbox.restore(); + OfficeChatTelemetryData.requestData = {}; + }); + + it("constructor", () => { + sandbox.stub(Correlator, "getId").returns("testCorrelationId"); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + const telemetryDataProperties = officeChatTelemetryData.telemetryData.properties; + chai.assert.equal(telemetryDataProperties[TelemetryProperty.CopilotChatCommand], "testCommand"); + chai.assert.equal( + telemetryDataProperties[TelemetryProperty.CopilotChatRequestId], + "testRequestId" + ); + chai.assert.equal( + telemetryDataProperties[TelemetryProperty.TriggerFrom], + TelemetryTriggerFrom.CopilotChat + ); + chai.assert.equal( + telemetryDataProperties[TelemetryProperty.CorrelationId], + "testCorrelationId" + ); + chai.assert.equal( + telemetryDataProperties[TelemetryProperty.CopilotChatParticipantId], + "testParticipantId" + ); + + chai.assert.equal(officeChatTelemetryData.command, "testCommand"); + chai.assert.equal(officeChatTelemetryData.requestId, "testRequestId"); + chai.assert.equal(officeChatTelemetryData.startTime, 0); + chai.assert.equal(officeChatTelemetryData.participantId, "testParticipantId"); + chai.assert.equal(officeChatTelemetryData.hasComplete, false); + chai.assert.equal(officeChatTelemetryData.hostType, ""); + chai.assert.equal(officeChatTelemetryData.relatedSampleName, ""); + chai.assert.equal(officeChatTelemetryData.timeToFirstToken, -1); + + chai.assert.equal( + OfficeChatTelemetryData.requestData["testRequestId"], + officeChatTelemetryData + ); + }); + + it("properties", () => { + sandbox.stub(Correlator, "getId").returns("testCorrelationId"); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + const properties = officeChatTelemetryData.properties; + + chai.assert.equal(properties[TelemetryProperty.CopilotChatCommand], "testCommand"); + chai.assert.equal(properties[TelemetryProperty.CopilotChatRequestId], "testRequestId"); + chai.assert.equal(properties[TelemetryProperty.TriggerFrom], TelemetryTriggerFrom.CopilotChat); + chai.assert.equal(properties[TelemetryProperty.CorrelationId], "testCorrelationId"); + chai.assert.equal(properties[TelemetryProperty.CopilotChatParticipantId], "testParticipantId"); + chai.assert.equal(officeChatTelemetryData.hostType, ""); + chai.assert.equal(officeChatTelemetryData.relatedSampleName, ""); + chai.assert.equal(officeChatTelemetryData.timeToFirstToken, -1); + }); + + describe("measurements", () => { + afterEach(() => { + sandbox.restore(); + OfficeChatTelemetryData.requestData = {}; + }); + + it("after init", () => { + sandbox.stub(Correlator, "getId").returns("testCorrelationId"); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + const measurements = officeChatTelemetryData.measurements; + + chai.assert.equal(Object.keys(measurements).length, 0); + }); + + it("after complete", () => { + sandbox.stub(Correlator, "getId").returns("testCorrelationId"); + sandbox.stub(performance, "now").returns(100); + sandbox.stub(utils, "countMessagesTokens").returns(200); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + officeChatTelemetryData.markComplete(); + + const measurements = officeChatTelemetryData.measurements; + + chai.assert.equal(measurements[TelemetryProperty.CopilotRequestChatCount], 200); + chai.assert.equal(measurements[TelemetryProperty.CopilotResponseChatCount], 200); + chai.assert.equal(measurements[TelemetryProperty.CopilotChatTimeToComplete], 0.1); + chai.assert.equal(measurements[TelemetryProperty.CopilotChatTimeToFirstToken], -1); + chai.assert.equal(measurements[TelemetryProperty.CopilotResponseChatCountPerSecond], 2000); + chai.assert.equal(measurements[TelemetryProperty.CopilotRequestChatCountPerSecond], 2000); + }); + }); + + it("createByParticipant", () => { + sandbox.stub(performance, "now").returns(100); + sandbox.stub(coreTools, "getUuid").returns("testRequestId"); + + const officeChatTelemetryData = OfficeChatTelemetryData.createByParticipant( + "testParticipantId", + "testCommand" + ); + + chai.assert.equal(officeChatTelemetryData.command, "testCommand"); + chai.assert.equal(officeChatTelemetryData.participantId, "testParticipantId"); + chai.assert.equal(officeChatTelemetryData.startTime, 100); + chai.assert.equal(officeChatTelemetryData.requestId, "testRequestId"); + }); + + describe("get", () => { + afterEach(() => { + sandbox.restore(); + OfficeChatTelemetryData.requestData = {}; + }); + + it("unknow requestId", () => { + chai.assert.isUndefined(OfficeChatTelemetryData.get("unknowRequestId")); + }); + + it("known requestId", () => { + sandbox.stub(Correlator, "getId").returns("testCorrelationId"); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + chai.assert.equal(OfficeChatTelemetryData.get("testRequestId"), officeChatTelemetryData); + }); + }); + + it("extendBy", () => { + const officeChatTelemetryData = OfficeChatTelemetryData.createByParticipant( + "testParticipantId", + "testCommand" + ); + + officeChatTelemetryData.extendBy({ testProperty: "testValue" }, { testMeasurement: 1 }); + + chai.assert.equal(officeChatTelemetryData.properties["testProperty"], "testValue"); + chai.assert.equal(officeChatTelemetryData.measurements["testMeasurement"], 1); + }); + + it("markComplete", () => { + sandbox.stub(utils, "countMessagesTokens").returns(100); + sandbox.stub(performance, "now").returns(100); + const officeChatTelemetryData = new OfficeChatTelemetryData( + "testCommand", + "testRequestId", + 0, + "testParticipantId" + ); + + chai.assert.equal(officeChatTelemetryData.hasComplete, false); + + officeChatTelemetryData.markComplete(); + + chai.assert.equal(officeChatTelemetryData.hasComplete, true); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[TelemetryProperty.CopilotRequestChatCount], + 100 + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[ + TelemetryProperty.CopilotResponseChatCount + ], + 100 + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[ + TelemetryProperty.CopilotChatTimeToComplete + ], + 0.1 + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.properties[TelemetryProperty.Success], + TelemetrySuccess.Yes + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.properties[TelemetryProperty.CopilotChatCompleteType], + "success" + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.properties[TelemetryProperty.HostType], + "" + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.properties[ + TelemetryProperty.CopilotChatRelatedSampleName + ], + "" + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[ + TelemetryProperty.CopilotChatTimeToFirstToken + ], + -1 + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[ + TelemetryProperty.CopilotRequestChatCountPerSecond + ], + 1000 + ); + chai.assert.equal( + officeChatTelemetryData.telemetryData.measurements[ + TelemetryProperty.CopilotResponseChatCountPerSecond + ], + 1000 + ); + + officeChatTelemetryData.markComplete("unsupportedPrompt"); + chai.assert.equal( + officeChatTelemetryData.telemetryData.properties[TelemetryProperty.CopilotChatCompleteType], + "success" + ); + }); +}); diff --git a/packages/vscode-extension/test/officeChat/utils.test.ts b/packages/vscode-extension/test/officeChat/utils.test.ts index ca95c858d4..de6247d99d 100644 --- a/packages/vscode-extension/test/officeChat/utils.test.ts +++ b/packages/vscode-extension/test/officeChat/utils.test.ts @@ -20,7 +20,7 @@ describe("File: officeChat/utils.ts", () => { beforeEach(() => { officeChatTelemetryDataMock = sandbox.createStubInstance(OfficeChatTelemetryData); officeChatTelemetryDataMock.chatMessages = []; - officeChatTelemetryDataMock.responseTokensPerRequest = []; + officeChatTelemetryDataMock.responseChatMessages = []; }); afterEach(() => { @@ -57,7 +57,7 @@ describe("File: officeChat/utils.ts", () => { }); officeChatTelemetryDataMock = sandbox.createStubInstance(OfficeChatTelemetryData); officeChatTelemetryDataMock.chatMessages = []; - officeChatTelemetryDataMock.responseTokensPerRequest = []; + officeChatTelemetryDataMock.responseChatMessages = []; }); afterEach(() => { sandbox.restore(); @@ -90,7 +90,7 @@ describe("File: officeChat/utils.ts", () => { const token = new CancellationToken(); const officeChatTelemetryDataMock = sandbox.createStubInstance(OfficeChatTelemetryData); officeChatTelemetryDataMock.chatMessages = []; - officeChatTelemetryDataMock.responseTokensPerRequest = []; + officeChatTelemetryDataMock.responseChatMessages = []; try { await utils.isInputHarmful( { prompt: "test" } as unknown as vscode.ChatRequest, @@ -108,7 +108,7 @@ describe("File: officeChat/utils.ts", () => { const token = new CancellationToken(); const officeChatTelemetryDataMock = sandbox.createStubInstance(OfficeChatTelemetryData); officeChatTelemetryDataMock.chatMessages = []; - officeChatTelemetryDataMock.responseTokensPerRequest = []; + officeChatTelemetryDataMock.responseChatMessages = []; try { await utils.isInputHarmful( { prompt: "test" } as unknown as vscode.ChatRequest,