test: update ai bot test cases (#10568)

OfficeDev · Dec 21, 2023 · cffd697 · cffd697
1 parent 3a5b1d1
commit cffd697
Show file tree

Hide file tree

Showing 10 changed files with 84 additions and 37 deletions.
diff --git a/packages/tests/src/ui-test/localdebug/localdebug-aiassistant-bot-ts.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-aiassistant-bot-ts.test.ts
@@ -67,6 +67,9 @@ describe("Local Debug Tests", function () {
       );
       await localDebugTestContext.validateLocalStateForBot();
       await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
         expectedWelcomeMessage:
           ValidationContent.AiAssistantBotWelcomeInstruction,
         expectedReplyMessage: ValidationContent.AiBotErrorMessage,

diff --git a/packages/tests/src/ui-test/localdebug/localdebug-aiassistant-bot.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-aiassistant-bot.test.ts
@@ -67,6 +67,9 @@ describe("Local Debug Tests", function () {
       );
       await localDebugTestContext.validateLocalStateForBot();
       await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
         expectedWelcomeMessage:
           ValidationContent.AiAssistantBotWelcomeInstruction,
         expectedReplyMessage: ValidationContent.AiBotErrorMessage,

diff --git a/packages/tests/src/ui-test/localdebug/localdebug-aichat-bot-ts.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-aichat-bot-ts.test.ts
@@ -15,6 +15,7 @@ import {
   Timeout,
   LocalDebugTaskLabel,
   DebugItemSelect,
+  ValidationContent,
 } from "../../utils/constants";
 import { Env } from "../../utils/env";
 import { it } from "../../utils/it";
@@ -62,7 +63,13 @@ describe("Local Debug Tests", function () {
         Env.password
       );
       await localDebugTestContext.validateLocalStateForBot();
-      await validateWelcomeAndReplyBot(page);
+      await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
+        expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+        expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+      });
     }
   );
 });
diff --git a/packages/tests/src/ui-test/localdebug/localdebug-aichat-bot.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-aichat-bot.test.ts
@@ -15,6 +15,7 @@ import {
   Timeout,
   LocalDebugTaskLabel,
   DebugItemSelect,
+  ValidationContent,
 } from "../../utils/constants";
 import { Env } from "../../utils/env";
 import { it } from "../../utils/it";
@@ -62,7 +63,13 @@ describe("Local Debug Tests", function () {
         Env.password
       );
       await localDebugTestContext.validateLocalStateForBot();
-      await validateWelcomeAndReplyBot(page);
+      await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
+        expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+        expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+      });
     }
   );
 });
diff --git a/packages/tests/src/ui-test/remotedebug/remotedebug-aiassistant-bot-ts-win-only.test.ts b/packages/tests/src/ui-test/remotedebug/remotedebug-aiassistant-bot-ts-win-only.test.ts
@@ -86,6 +86,9 @@ describe("Remote debug Tests", function () {
       );
       await driver.sleep(Timeout.longTimeWait);
       await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
         expectedWelcomeMessage:
           ValidationContent.AiAssistantBotWelcomeInstruction,
         expectedReplyMessage: ValidationContent.AiBotErrorMessage,

diff --git a/packages/tests/src/ui-test/remotedebug/remotedebug-aiassistant-bot-win-only.test.ts b/packages/tests/src/ui-test/remotedebug/remotedebug-aiassistant-bot-win-only.test.ts
@@ -86,6 +86,9 @@ describe("Remote debug Tests", function () {
       );
       await driver.sleep(Timeout.longTimeWait);
       await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
         expectedWelcomeMessage:
           ValidationContent.AiAssistantBotWelcomeInstruction,
         expectedReplyMessage: ValidationContent.AiBotErrorMessage,

diff --git a/packages/tests/src/ui-test/remotedebug/remotedebug-aichat-bot-ts-win-only.test.ts b/packages/tests/src/ui-test/remotedebug/remotedebug-aichat-bot-ts-win-only.test.ts
@@ -6,7 +6,7 @@
  */
 import * as path from "path";
 import { VSBrowser } from "vscode-extension-tester";
-import { Timeout } from "../../utils/constants";
+import { Timeout, ValidationContent } from "../../utils/constants";
 import {
   RemoteDebugTestContext,
   runProvision,
@@ -82,7 +82,13 @@ describe("Remote debug Tests", function () {
         Env.password
       );
       await driver.sleep(Timeout.longTimeWait);
-      await validateWelcomeAndReplyBot(page);
+      await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
+        expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+        expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+      });
     }
   );
 });
diff --git a/packages/tests/src/ui-test/remotedebug/remotedebug-aichat-bot-win-only.test.ts b/packages/tests/src/ui-test/remotedebug/remotedebug-aichat-bot-win-only.test.ts
@@ -6,7 +6,7 @@
  */
 import * as path from "path";
 import { VSBrowser } from "vscode-extension-tester";
-import { Timeout } from "../../utils/constants";
+import { Timeout, ValidationContent } from "../../utils/constants";
 import {
   RemoteDebugTestContext,
   runProvision,
@@ -82,7 +82,13 @@ describe("Remote debug Tests", function () {
         Env.password
       );
       await driver.sleep(Timeout.longTimeWait);
-      await validateWelcomeAndReplyBot(page);
+      await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
+        expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+        expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+      });
     }
   );
 });
diff --git a/packages/tests/src/ui-test/treeview/treeviewContext.ts b/packages/tests/src/ui-test/treeview/treeviewContext.ts
@@ -63,19 +63,19 @@ export async function createSampleProject(
   await webView.switchToFrame();
   let foundSample = false;
   console.log("finding sample...");
-  const elements = await webView.findWebElements(
-    By.xpath('//div[@class="sample-gallery"]//div[@class="sample-list-item"]')
-  );
+
+  const elements = await webView.findWebElements(By.css("h3"));
   for (const element of elements) {
-    const sampleItem = await element.findElement(By.css("h3"));
-    const SampleItemName = await sampleItem.getText();
+    const SampleItemName = await element.getText();
     if (SampleItemName === sample) {
       foundSample = true;
       await element.click();
       break;
     }
   }
+
   if (!foundSample) {
+    await VSBrowser.instance.takeScreenshot(getScreenshotName("no-sample-"));
     throw new Error(`Not found sample ${sample}`);
   }
   const button = await webView.findWebElement(

diff --git a/packages/tests/src/utils/playwrightOperation.ts b/packages/tests/src/utils/playwrightOperation.ts
@@ -989,10 +989,14 @@ export async function validateEchoBot(
 export async function validateWelcomeAndReplyBot(
   page: Page,
   options: {
+    hasWelcomeMessage?: boolean;
+    hasCommandReplyValidation: boolean;
     botCommand?: string;
     expectedWelcomeMessage?: string;
     expectedReplyMessage?: string;
   } = {
+    hasWelcomeMessage: true,
+    hasCommandReplyValidation: true,
     botCommand: "helloWorld",
     expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
     expectedReplyMessage: ValidationContent.AiBotErrorMessage,
@@ -1017,34 +1021,39 @@ export async function validateWelcomeAndReplyBot(
       console.log("no message to dismiss");
     }
 
-    await RetryHandler.retry(async () => {
-      await frame?.waitForSelector(
-        `p:has-text("${
+    if (options.hasWelcomeMessage) {
+      await RetryHandler.retry(async () => {
+        await frame?.waitForSelector(
+          `p:has-text("${
+            options?.expectedWelcomeMessage ||
+            ValidationContent.AiChatBotWelcomeInstruction
+          }")`
+        );
+        console.log(
           options?.expectedWelcomeMessage ||
-          ValidationContent.AiChatBotWelcomeInstruction
-        }")`
-      );
-      console.log(
-        options?.expectedWelcomeMessage ||
-          ValidationContent.AiChatBotWelcomeInstruction
-      );
-      console.log("verified bot that it has sent welcome!!!");
-    }, 2);
+            ValidationContent.AiChatBotWelcomeInstruction
+        );
+        console.log("verified bot that it has sent welcome!!!");
+      }, 2);
+    }
+
+    if (options.hasCommandReplyValidation) {
+      await RetryHandler.retry(async () => {
+        console.log("sending message ", options?.botCommand || "helloWorld");
+        await frame?.fill(
+          'div.ck-content[role="textbox"]',
+          options?.botCommand || "helloWorld"
+        );
+        await frame?.click('button[name="send"]');
+        await frame?.waitForSelector(
+          `p:has-text("${options?.expectedReplyMessage}")`
+        );
+        console.log(
+          `verify bot successfully with content ${options?.expectedReplyMessage}!!!`
+        );
+      }, 2);
+    }
 
-    await RetryHandler.retry(async () => {
-      console.log("sending message ", options?.botCommand || "helloWorld");
-      await frame?.fill(
-        'div.ck-content[role="textbox"]',
-        options?.botCommand || "helloWorld"
-      );
-      await frame?.click('button[name="send"]');
-      await frame?.waitForSelector(
-        `p:has-text("${options?.expectedReplyMessage}")`
-      );
-      console.log(
-        `verify bot successfully with content ${options?.expectedReplyMessage}!!!`
-      );
-    }, 2);
     await page.waitForTimeout(Timeout.shortTimeLoading);
   } catch (error) {
     await page.screenshot({