fix: add telemetry for office agent for unit test - resolve comments

OfficeDev · Jun 26, 2024 · 3b6a02a · 3b6a02a
1 parent d1d0637
commit 3b6a02a
Show file tree

Hide file tree

Showing 24 changed files with 397 additions and 190 deletions.
diff --git a/packages/vscode-extension/src/officeChat/commands/create/helper.ts b/packages/vscode-extension/src/officeChat/commands/create/helper.ts
@@ -25,6 +25,7 @@ import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../
 import { OfficeXMLAddinGenerator } from "./officeXMLAddinGenerator/generator";
 import { CreateProjectInputs } from "@microsoft/teamsfx-api";
 import { core } from "../../../globalVariables";
+import { ProjectMiniData } from "../../types";
 
 export async function matchOfficeProject(
   request: ChatRequest,
@@ -39,8 +40,8 @@ export async function matchOfficeProject(
     getOfficeProjectMatchSystemPrompt(allOfficeProjectMetadata),
     new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
   ];
-  const t0 = performance.now();
   let response = "";
+  telemetryData.chatMessages.push(...messages);
   try {
     response = await getCopilotResponseAsString("copilot-gpt-4", messages, token);
   } catch (error) {
@@ -50,12 +51,7 @@ export async function matchOfficeProject(
       telemetryData.markComplete("unsupportedPrompt");
     }
   }
-  const t1 = performance.now();
-  telemetryData.responseTokensPerRequest.push(
-    OfficeChatTelemetryData.calculateResponseTokensPerRequest(response, t0, t1)
-  );
-  telemetryData.chatMessages.push(
-    ...messages,
+  telemetryData.responseChatMessages.push(
     new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, response)
   );
   let matchedProjectId: string;
@@ -112,7 +108,7 @@ export function getOfficeTemplateMetadata(): ProjectMetadata[] {
 export async function showOfficeSampleFileTree(
   projectMetadata: ProjectMetadata,
   response: ChatResponseStream
-): Promise<object> {
+): Promise<ProjectMiniData> {
   response.markdown(
     "\nWe've found a sample project that matches your description. Take a look at it below."
   );
@@ -128,7 +124,11 @@ export async function showOfficeSampleFileTree(
     20
   );
   response.filetree(nodes, Uri.file(path.join(tempFolder, downloadUrlInfo.dir)));
-  return { path: path.join(tempFolder, downloadUrlInfo.dir), host: host };
+  const result: ProjectMiniData = {
+    path: path.join(tempFolder, downloadUrlInfo.dir),
+    host: host,
+  };
+  return result;
 }
 
 export async function showOfficeTemplateFileTree(

diff --git a/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts b/packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts
@@ -13,7 +13,7 @@ import {
 import { OfficeChatCommand, officeChatParticipantId } from "../../consts";
 import { verbatimCopilotInteraction } from "../../../chat/utils";
 import { isInputHarmful } from "../../utils";
-import { ICopilotChatOfficeResult } from "../../types";
+import { ICopilotChatOfficeResult, ProjectMiniData } from "../../types";
 import { describeOfficeProjectSystemPrompt } from "../../officePrompts";
 import { TelemetryEvent } from "../../../telemetry/extTelemetryEvents";
 import { ExtTelemetry } from "../../../telemetry/extTelemetry";
@@ -22,6 +22,7 @@ import { localize } from "../../../utils/localizeUtils";
 import { Planner } from "../../common/planner";
 import { CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID } from "../../consts";
 import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../../telemetry";
+import { ProjectMetadata } from "../../../chat/commands/create/types";
 
 export default async function officeCreateCommandHandler(
   request: ChatRequest,
@@ -78,26 +79,27 @@ export default async function officeCreateCommandHandler(
       );
 
       if (matchedResult.type === "sample") {
-        const sampleInfos = await showOfficeSampleFileTree(matchedResult, response);
-        const folder = (sampleInfos as any)?.["path"];
-        const hostType = (sampleInfos as any)?.["host"].toLowerCase();
+        const sampleInfos: ProjectMiniData = await showOfficeSampleFileTree(
+          matchedResult,
+          response
+        );
+        const folder = sampleInfos.path;
+        const hostType = sampleInfos.host.toLowerCase();
         const sampleTitle = localize("teamstoolkit.chatParticipants.create.sample");
         officeChatTelemetryData.setHostType(hostType);
-        const matchResultInfo = "sample";
         response.button({
           command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
-          arguments: [folder, officeChatTelemetryData.requestId, matchResultInfo],
+          arguments: [folder, officeChatTelemetryData.requestId, matchedResult.type],
           title: sampleTitle,
         });
       } else {
         const tmpHostType = (matchedResult.data as any)?.["addin-host"].toLowerCase();
         const tmpFolder = await showOfficeTemplateFileTree(matchedResult.data as any, response);
         const templateTitle = localize("teamstoolkit.chatParticipants.create.template");
         officeChatTelemetryData.setHostType(tmpHostType);
-        const tmpmatchResultInfo = "template";
         response.button({
           command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
-          arguments: [tmpFolder, officeChatTelemetryData.requestId, tmpmatchResultInfo],
+          arguments: [tmpFolder, officeChatTelemetryData.requestId, matchedResult.type],
           title: templateTitle,
         });
       }

diff --git a/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts b/packages/vscode-extension/src/officeChat/commands/nextStep/officeNextstepCommandHandler.ts
@@ -75,13 +75,8 @@ export default async function officeNextStepCommandHandler(
     if (s.description instanceof Function) {
       s.description = s.description(status);
     }
-    const t0 = performance.now();
     const stepDescription = await describeOfficeStep(s, token, officeChatTelemetryData);
-    const t1 = performance.now();
-    officeChatTelemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(stepDescription, t0, t1)
-    );
-    officeChatTelemetryData.chatMessages.push(
+    officeChatTelemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, stepDescription)
     );
     const title = s.docLink ? `[${s.title}](${s.docLink})` : s.title;

diff --git a/packages/vscode-extension/src/officeChat/common/planner.ts b/packages/vscode-extension/src/officeChat/common/planner.ts
@@ -86,8 +86,7 @@ export class Planner {
 ${localize("teamstoolkit.chatParticipants.officeAddIn.printer.outputTemplate.intro")}\n
 ${purified}
 `);
-    const spec = new Spec(purified);
-    spec.appendix.telemetryData.requestId = telemetryData.requestId;
+    const spec = new Spec(purified, telemetryData.requestId);
     try {
       for (let index = 0; index < candidates.length; index++) {
         const candidate = candidates[index];
@@ -147,14 +146,11 @@ ${purified}
     telemetryData.setHostType(spec.appendix.host?.toLowerCase());
     telemetryData.setTimeToFirstToken(spec.appendix.telemetryData.timeToFirstToken);
     telemetryData.setRelatedSampleName(spec.appendix.telemetryData.relatedSampleName.toString());
-    // telemetryData.setCodeClassAndMembers(
-    //   spec.appendix.telemetryData.codeClassAndMembers.toString()
-    // );
     for (const chatMessage of spec.appendix.telemetryData.chatMessages) {
       telemetryData.chatMessages.push(chatMessage);
     }
-    for (const responseTokensPerRequest of spec.appendix.telemetryData.responseTokensPerRequest) {
-      telemetryData.responseTokensPerRequest.push(responseTokensPerRequest);
+    for (const responseChatMessage of spec.appendix.telemetryData.responseChatMessages) {
+      telemetryData.responseChatMessages.push(responseChatMessage);
     }
     const debugInfo = `
       ## Time cost:\n

diff --git a/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts b/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts
@@ -104,14 +104,9 @@ export class SampleProvider {
     }
 
     countOfLLMInvoke += 1;
-    const timeStart = performance.now();
     const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token);
-    const timeEnd = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, timeStart, timeEnd)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      sampleMessage,
+    spec.appendix.telemetryData.chatMessages.push(sampleMessage);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     const returnObject: { picked: string[] } = JSON.parse(
@@ -301,14 +296,9 @@ export class SampleProvider {
       LanguageModelChatMessageRole.User,
       getMoreRelevantMethodsOrPropertiesPrompt
     );
-    const t0 = performance.now();
     const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token);
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      sampleMessage,
+    spec.appendix.telemetryData.chatMessages.push(sampleMessage);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     let returnObject: { picked: string[] } = { picked: [] };

diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts b/packages/vscode-extension/src/officeChat/common/skills/codeExplainer.ts
@@ -57,18 +57,13 @@ Let's think it step by step.
       new LanguageModelChatMessage(LanguageModelChatMessageRole.User, systemPrompt),
       new LanguageModelChatMessage(LanguageModelChatMessageRole.User, userPrompt),
     ];
-    const t0 = performance.now();
     const copilotResponse = await getCopilotResponseAsString(
       "copilot-gpt-3.5-turbo",
       messages,
       token
     );
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      ...messages,
+    spec.appendix.telemetryData.chatMessages.push(...messages);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
 

diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts b/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts
@@ -212,18 +212,13 @@ ${spec.appendix.codeExplanation
       new LanguageModelChatMessage(LanguageModelChatMessageRole.User, userPrompt),
       new LanguageModelChatMessage(LanguageModelChatMessageRole.System, defaultSystemPrompt),
     ];
-    const t0 = performance.now();
     let copilotResponse = await getCopilotResponseAsString(
       "copilot-gpt-3.5-turbo", // "copilot-gpt-4", // "copilot-gpt-3.5-turbo",
       messages,
       token
     );
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      ...messages,
+    spec.appendix.telemetryData.chatMessages.push(...messages);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     let copilotRet: {
@@ -329,18 +324,13 @@ ${spec.appendix.codeExplanation
       );
     }
 
-    const t0 = performance.now();
     let copilotResponse = await getCopilotResponseAsString(
       "copilot-gpt-4", //"copilot-gpt-4", // "copilot-gpt-3.5-turbo",
       messages,
       token
     );
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      ...messages,
+    spec.appendix.telemetryData.chatMessages.push(...messages);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     let copilotRet: {
@@ -500,14 +490,9 @@ ${spec.appendix.codeExplanation
     }
     console.debug(`token count: ${msgCount}, number of messages remains: ${messages.length}.`);
 
-    const t0 = performance.now();
     const copilotResponse = await getCopilotResponseAsString(model, messages, token);
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      ...messages,
+    spec.appendix.telemetryData.chatMessages.push(...messages);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     // extract the code snippet and the api list out

diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts b/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts
@@ -353,14 +353,9 @@ export class CodeIssueCorrector implements ISkill {
       msgCount = countMessagesTokens(messages);
     }
     // console.debug(`token count: ${msgCount}, number of messages remains: ${messages.length}.`);
-    const t0 = performance.now();
     const copilotResponse = await getCopilotResponseAsString(model, messages, token);
-    const t1 = performance.now();
-    spec.appendix.telemetryData.responseTokensPerRequest.push(
-      OfficeChatTelemetryData.calculateResponseTokensPerRequest(copilotResponse, t0, t1)
-    );
-    spec.appendix.telemetryData.chatMessages.push(
-      ...messages,
+    spec.appendix.telemetryData.chatMessages.push(...messages);
+    spec.appendix.telemetryData.responseChatMessages.push(
       new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, copilotResponse)
     );
     // extract the code snippet

diff --git a/packages/vscode-extension/src/officeChat/common/skills/spec.ts b/packages/vscode-extension/src/officeChat/common/skills/spec.ts
@@ -23,18 +23,17 @@ export class Spec {
       requestId: string;
       isHarmful: boolean;
       relatedSampleName: string[];
-      codeClassAndMembers: string[];
       timeToFirstToken: DOMHighResTimeStamp;
       chatMessages: LanguageModelChatMessage[];
-      responseTokensPerRequest: number[];
+      responseChatMessages: LanguageModelChatMessage[];
       properties: { [key: string]: string };
       measurements: { [key: string]: number };
     };
     complexity: number;
     shouldContinue: boolean;
   };
 
-  constructor(userInput: string) {
+  constructor(userInput: string, requestId?: string) {
     this.userInput = userInput;
     this.taskSummary = "";
     this.sections = [];
@@ -49,13 +48,12 @@ export class Spec {
       apiDeclarationsReference: new Map<string, SampleData>(),
       isCustomFunction: false,
       telemetryData: {
-        requestId: "",
+        requestId: requestId ? requestId : "",
         isHarmful: false,
         relatedSampleName: [],
-        codeClassAndMembers: [],
         timeToFirstToken: 0,
         chatMessages: [],
-        responseTokensPerRequest: [],
+        responseChatMessages: [],
         properties: {},
         measurements: {},
       },