refactor: add telemetry for office agent

packages/vscode-extension/src/officeChat/commands/create/helper.ts

@@ -14,22 +14,23 @@ import {
   LanguageModelChatMessage,
   LanguageModelChatMessageRole,
 } from "vscode";
-import { IChatTelemetryData } from "../../../chat/types";
 import { ProjectMetadata } from "../../../chat/commands/create/types";
 import { getCopilotResponseAsString } from "../../../chat/utils";
 import { getOfficeProjectMatchSystemPrompt } from "../../officePrompts";
 import { officeSampleProvider } from "./officeSamples";
 import { fileTreeAdd, buildFileTree } from "../../../chat/commands/create/helper";
-import { getOfficeSampleDownloadUrlInfo } from "../../utils";
+import { getOfficeSample } from "../../utils";
 import { getSampleFileInfo } from "@microsoft/teamsfx-core/build/component/generator/utils";
+import { OfficeChatTelemetryData } from "../../telemetry";
 import { OfficeXMLAddinGenerator } from "./officeXMLAddinGenerator/generator";
 import { CreateProjectInputs } from "@microsoft/teamsfx-api";
 import { core } from "../../../globalVariables";
+import { OfficeProjectInfo } from "../../types";
 
 export async function matchOfficeProject(
   request: ChatRequest,
   token: CancellationToken,
-  telemetryMetadata: IChatTelemetryData
+  telemetryData: OfficeChatTelemetryData
 ): Promise<ProjectMetadata | undefined> {
   const allOfficeProjectMetadata = [
     ...getOfficeTemplateMetadata(),
@@ -39,8 +40,12 @@ export async function matchOfficeProject(
     getOfficeProjectMatchSystemPrompt(allOfficeProjectMetadata),
     new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
   ];
-  telemetryMetadata.chatMessages.push(...messages);
-  const response = await getCopilotResponseAsString("copilot-gpt-4", messages, token);
+  let response = "";
+  telemetryData.chatMessages.push(...messages);
+  response = await getCopilotResponseAsString("copilot-gpt-4", messages, token);
+  telemetryData.responseChatMessages.push(
+    new LanguageModelChatMessage(LanguageModelChatMessageRole.Assistant, response)
+  );
   let matchedProjectId: string;
   if (response) {
     try {
@@ -95,23 +100,27 @@ export function getOfficeTemplateMetadata(): ProjectMetadata[] {
 export async function showOfficeSampleFileTree(
   projectMetadata: ProjectMetadata,
   response: ChatResponseStream
-): Promise<string> {
+): Promise<OfficeProjectInfo> {
   response.markdown(
     "\nWe've found a sample project that matches your description. Take a look at it below."
   );
-  const downloadUrlInfo = await getOfficeSampleDownloadUrlInfo(projectMetadata.id);
-  const { samplePaths, fileUrlPrefix } = await getSampleFileInfo(downloadUrlInfo, 2);
+  const sample = await getOfficeSample(projectMetadata.id);
+  const { samplePaths, fileUrlPrefix } = await getSampleFileInfo(sample.downloadUrlInfo, 2);
   const tempFolder = tmp.dirSync({ unsafeCleanup: true }).name;
   const nodes = await buildFileTree(
     fileUrlPrefix,
     samplePaths,
     tempFolder,
-    downloadUrlInfo.dir,
+    sample.downloadUrlInfo.dir,
     2,
     20
   );
-  response.filetree(nodes, Uri.file(path.join(tempFolder, downloadUrlInfo.dir)));
-  return path.join(tempFolder, downloadUrlInfo.dir);
+  response.filetree(nodes, Uri.file(path.join(tempFolder, sample.downloadUrlInfo.dir)));
+  const result: OfficeProjectInfo = {
+    path: path.join(tempFolder, sample.downloadUrlInfo.dir),
+    host: sample.types[0],
+  };
+  return result;
 }
 
 export async function showOfficeTemplateFileTree(

packages/vscode-extension/src/officeChat/commands/create/officeCreateCommandHandler.ts

@@ -13,23 +13,23 @@
 import { OfficeChatCommand, officeChatParticipantId } from "../../consts";
 import { verbatimCopilotInteraction } from "../../../chat/utils";
 import { isInputHarmful } from "../../utils";
-import { ICopilotChatOfficeResult } from "../../types";
+import { ICopilotChatOfficeResult, OfficeProjectInfo } from "../../types";
 import { describeOfficeProjectSystemPrompt } from "../../officePrompts";
 import { TelemetryEvent } from "../../../telemetry/extTelemetryEvents";
 import { ExtTelemetry } from "../../../telemetry/extTelemetry";
-import { ChatTelemetryData } from "../../../chat/telemetry";
 import { matchOfficeProject, showOfficeSampleFileTree, showOfficeTemplateFileTree } from "./helper";
 import { localize } from "../../../utils/localizeUtils";
 import { Planner } from "../../common/planner";
 import { CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID } from "../../consts";
+import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../../telemetry";
 
 export default async function officeCreateCommandHandler(
   request: ChatRequest,
   context: ChatContext,
   response: ChatResponseStream,
   token: CancellationToken
 ): Promise<ICopilotChatOfficeResult> {
-  const officeChatTelemetryData = ChatTelemetryData.createByParticipant(
+  const officeChatTelemetryData = OfficeChatTelemetryData.createByParticipant(
     officeChatParticipantId,
     OfficeChatCommand.Create
   );
@@ -39,9 +39,10 @@
   );
 
   if (request.prompt.trim() === "") {
+    officeChatTelemetryData.setTimeToFirstToken();
     response.markdown(localize("teamstoolkit.chatParticipants.officeAddIn.create.noPromptAnswer"));
-
-    officeChatTelemetryData.markComplete();
+    officeChatTelemetryData.setBlockReason(OfficeChatTelemetryBlockReasonEnum.UnsupportedInput);
+    officeChatTelemetryData.markComplete("fail");
     ExtTelemetry.sendTelemetryEvent(
       TelemetryEvent.CopilotChat,
       officeChatTelemetryData.properties,
@@ -54,67 +55,95 @@
       },
     };
   }
-
-  const isHarmful = await isInputHarmful(request, token);
+  const isHarmful = await isInputHarmful(request, token, officeChatTelemetryData);
   if (!isHarmful) {
-    const matchedResult = await matchOfficeProject(request, token, officeChatTelemetryData);
-    if (matchedResult) {
-      response.markdown(
-        localize("teamstoolkit.chatParticipants.officeAddIn.create.projectMatched")
-      );
-      const describeProjectChatMessages = [
-        describeOfficeProjectSystemPrompt(),
-        new LanguageModelChatMessage(
-          LanguageModelChatMessageRole.User,
-          `The project you are looking for is '${JSON.stringify(matchedResult)}'.`
-        ),
-      ];
-      officeChatTelemetryData.chatMessages.push(...describeProjectChatMessages);
-
-      await verbatimCopilotInteraction(
-        "copilot-gpt-3.5-turbo",
-        describeProjectChatMessages,
-        response,
-        token
-      );
-      if (matchedResult.type === "sample") {
-        const folder = await showOfficeSampleFileTree(matchedResult, response);
-        const sampleTitle = localize("teamstoolkit.chatParticipants.create.sample");
-        response.button({
-          command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
-          arguments: [folder],
-          title: sampleTitle,
-        });
+    try {
+      const matchedResult = await matchOfficeProject(request, token, officeChatTelemetryData);
+      if (matchedResult) {
+        officeChatTelemetryData.setTimeToFirstToken();
+        response.markdown(
+          localize("teamstoolkit.chatParticipants.officeAddIn.create.projectMatched")
+        );
+        const describeProjectChatMessages = [
+          describeOfficeProjectSystemPrompt(),
+          new LanguageModelChatMessage(
+            LanguageModelChatMessageRole.User,
+            `The project you are looking for is '${JSON.stringify(matchedResult)}'.`
+          ),
+        ];
+        officeChatTelemetryData.chatMessages.push(...describeProjectChatMessages);
+        await verbatimCopilotInteraction(
+          "copilot-gpt-3.5-turbo",
+          describeProjectChatMessages,
+          response,
+          token
+        );
+        if (matchedResult.type === "sample") {
+          const sampleInfos: OfficeProjectInfo = await showOfficeSampleFileTree(
+            matchedResult,
+            response
+          );
+          const folder = sampleInfos.path;
+          const hostType = sampleInfos.host.toLowerCase();
+          const sampleTitle = localize("teamstoolkit.chatParticipants.create.sample");
+          officeChatTelemetryData.setHostType(hostType);
+          response.button({
+            command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
+            arguments: [folder, officeChatTelemetryData.requestId, matchedResult.type],
+            title: sampleTitle,
+          });
+        } else {
+          const tmpHostType = (matchedResult.data as any)?.["addin-host"].toLowerCase();
+          const tmpFolder = await showOfficeTemplateFileTree(matchedResult.data, response);
+          const templateTitle = localize("teamstoolkit.chatParticipants.create.template");
+          officeChatTelemetryData.setHostType(tmpHostType);
+          response.button({
+            command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
+            arguments: [tmpFolder, officeChatTelemetryData.requestId, matchedResult.type],
+            title: templateTitle,
+          });
+        }
+        officeChatTelemetryData.markComplete();
+      } else {
+        let chatResult: ICopilotChatOfficeResult = {};
+        try {
+          chatResult = await Planner.getInstance().processRequest(
+            new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
+            request,
+            response,
+            token,
+            OfficeChatCommand.Create,
+            officeChatTelemetryData
+          );
+          officeChatTelemetryData.markComplete();
+        } catch (error) {
+          officeChatTelemetryData.markComplete("fail");
+        }
+        ExtTelemetry.sendTelemetryEvent(
+          TelemetryEvent.CopilotChat,
+          officeChatTelemetryData.properties,
+          officeChatTelemetryData.measurements
+        );
+        return chatResult;
+      }
+    } catch (error) {
+      if ((error as Error).message.includes("off_topic")) {
+        officeChatTelemetryData.setBlockReason(OfficeChatTelemetryBlockReasonEnum.OffTopic);
       } else {
-        const tmpFolder = await showOfficeTemplateFileTree(matchedResult.data, response);
-        const templateTitle = localize("teamstoolkit.chatParticipants.create.template");
-        response.button({
-          command: CHAT_CREATE_OFFICE_PROJECT_COMMAND_ID,
-          arguments: [tmpFolder],
-          title: templateTitle,
-        });
+        officeChatTelemetryData.setBlockReason(
+          OfficeChatTelemetryBlockReasonEnum.LanguageModelError
+        );
       }
-    } else {
-      const chatResult = await Planner.getInstance().processRequest(
-        new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
-        request,
-        response,
-        token,
-        OfficeChatCommand.Create,
-        officeChatTelemetryData
-      );
-      officeChatTelemetryData.markComplete();
-      ExtTelemetry.sendTelemetryEvent(
-        TelemetryEvent.CopilotChat,
-        officeChatTelemetryData.properties,
-        officeChatTelemetryData.measurements
-      );
-      return chatResult;
+      officeChatTelemetryData.setTimeToFirstToken();
+      response.markdown(localize("teamstoolkit.chatParticipants.officeAddIn.default.canNotAssist"));
+      officeChatTelemetryData.markComplete("fail");
     }
   } else {
+    officeChatTelemetryData.setTimeToFirstToken();
     response.markdown(localize("teamstoolkit.chatParticipants.officeAddIn.harmfulInputResponse"));
+    officeChatTelemetryData.setBlockReason(OfficeChatTelemetryBlockReasonEnum.RAI);
+    officeChatTelemetryData.markComplete("fail");
   }
-  officeChatTelemetryData.markComplete();
   ExtTelemetry.sendTelemetryEvent(
     TelemetryEvent.CopilotChat,
     officeChatTelemetryData.properties,

packages/vscode-extension/src/officeChat/commands/generatecode/generatecodeCommandHandler.ts

@@ -13,17 +13,17 @@
 import { localize } from "../../../utils/localizeUtils";
 import { OfficeChatCommand, officeChatParticipantId } from "../../consts";
 import { Planner } from "../../common/planner";
-import { ChatTelemetryData } from "../../../chat/telemetry";
 import { isInputHarmful } from "../../utils";
 import { ICopilotChatOfficeResult } from "../../types";
+import { OfficeChatTelemetryBlockReasonEnum, OfficeChatTelemetryData } from "../../telemetry";
 
 export default async function generatecodeCommandHandler(
   request: ChatRequest,
   context: ChatContext,
   response: ChatResponseStream,
   token: CancellationToken
 ): Promise<ICopilotChatOfficeResult> {
-  const officeChatTelemetryData = ChatTelemetryData.createByParticipant(
+  const officeChatTelemetryData = OfficeChatTelemetryData.createByParticipant(
     officeChatParticipantId,
     OfficeChatCommand.GenerateCode
   );
@@ -33,11 +33,12 @@
   );
 
   if (request.prompt.trim() === "") {
+    officeChatTelemetryData.setTimeToFirstToken();
     response.markdown(
       localize("teamstoolkit.chatParticipants.officeAddIn.generateCode.noPromptAnswer")
     );
-
-    officeChatTelemetryData.markComplete();
+    officeChatTelemetryData.setBlockReason(OfficeChatTelemetryBlockReasonEnum.UnsupportedInput);
+    officeChatTelemetryData.markComplete("fail");
     ExtTelemetry.sendTelemetryEvent(
       TelemetryEvent.CopilotChat,
       officeChatTelemetryData.properties,
@@ -66,26 +67,33 @@
     }
   }
 
-  const isHarmful = await isInputHarmful(request, token);
+  const isHarmful = await isInputHarmful(request, token, officeChatTelemetryData);
   if (!isHarmful) {
-    const chatResult = await Planner.getInstance().processRequest(
-      new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
-      request,
-      response,
-      token,
-      OfficeChatCommand.GenerateCode,
-      officeChatTelemetryData
-    );
-    officeChatTelemetryData.markComplete();
+    let chatResult: ICopilotChatOfficeResult = {};
+    try {
+      chatResult = await Planner.getInstance().processRequest(
+        new LanguageModelChatMessage(LanguageModelChatMessageRole.User, request.prompt),
+        request,
+        response,
+        token,
+        OfficeChatCommand.GenerateCode,
+        officeChatTelemetryData
+      );
+      officeChatTelemetryData.markComplete();
+    } catch (error) {
+      officeChatTelemetryData.markComplete("fail");
+    }
     ExtTelemetry.sendTelemetryEvent(
       TelemetryEvent.CopilotChat,
       officeChatTelemetryData.properties,
       officeChatTelemetryData.measurements
     );
     return chatResult;
   } else {
+    officeChatTelemetryData.setTimeToFirstToken();
     response.markdown(localize("teamstoolkit.chatParticipants.officeAddIn.harmfulInputResponse"));
-    officeChatTelemetryData.markComplete();
+    officeChatTelemetryData.setBlockReason(OfficeChatTelemetryBlockReasonEnum.RAI);
+    officeChatTelemetryData.markComplete("fail");
     ExtTelemetry.sendTelemetryEvent(
       TelemetryEvent.CopilotChat,
       officeChatTelemetryData.properties,