chore(ui): improve e2e playwright test stability (#1116)

src/leapfrogai_ui/src/lib/components/Message.test.ts

@@ -254,5 +254,12 @@ describe('Message component', () => {
       });
       screen.getByText(fakeAssistants[0].name!);
     });
+    it('shows a loading skeleton if the message text is empty', () => {
+      render(Message, {
+        ...getDefaultMessageProps(),
+        message: getFakeMessage({ role: 'assistant', content: '' })
+      });
+      expect(screen.getByTestId('loading-msg')).toBeInTheDocument();
+    });
   });
 });

src/leapfrogai_ui/testUtils/fakeData/index.ts

@@ -32,8 +32,11 @@ type FakeMessageOptions = {
   created_at?: number;
 };
 export const getFakeMessage = (options: FakeMessageOptions = {}): LFMessage => {
+  //allow empty string for content
+  if (options.content === undefined || options.content === null)
+    options.content = faker.lorem.lines(1);
   const messageContent: MessageContent[] = [
-    { type: 'text', text: { value: options.content || faker.lorem.lines(1), annotations: [] } }
+    { type: 'text', text: { value: options.content, annotations: [] } }
   ];
 
   const {

src/leapfrogai_ui/tests/assistant-avatars.test.ts

@@ -12,6 +12,7 @@ import {
   savePictogram,
   uploadAvatar
 } from './helpers/assistantHelpers';
+import { loadEditAssistantPage, loadNewAssistantPage } from './helpers/navigationHelpers';
 
 test.afterEach(async ({ openAIClient }) => {
   await deleteAllAssistants(openAIClient);
@@ -25,7 +26,7 @@ test('it can search for and choose a pictogram as an avatar', async ({ page }) =
   const assistantInput = getFakeAssistantInput();
   const pictogramName = getRandomPictogramName();
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await fillOutRequiredAssistantFields(assistantInput, page);
 
@@ -53,7 +54,7 @@ test('it can search for and choose a pictogram as an avatar', async ({ page }) =
 test('it can upload an image as an avatar', async ({ page }) => {
   const assistantInput = getFakeAssistantInput();
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await fillOutRequiredAssistantFields(assistantInput, page);
 
@@ -71,7 +72,7 @@ test('it can upload an image as an avatar', async ({ page }) => {
 });
 
 test('it can change an image uploaded as an avatar', async ({ page }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await page.getByTestId('mini-avatar-container').click();
   await uploadAvatar(page);
@@ -97,7 +98,7 @@ test('it shows an error when clicking save on the upload tab if no image is uplo
 }) => {
   const assistantInput = getFakeAssistantInput();
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await fillOutRequiredAssistantFields(assistantInput, page);
 
@@ -114,7 +115,7 @@ test('it shows an error when clicking save on the upload tab if no image is uplo
 // Note - not testing too large file size validation because we would have to store a large file just for a test
 
 test('it removes an uploaded image and keeps the original pictogram on save', async ({ page }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByTestId('mini-avatar-container').click();
 
   await uploadAvatar(page);
@@ -135,7 +136,7 @@ test('it removes an uploaded image and keeps the original pictogram on save', as
 test('it keeps the original pictogram on cancel after uploading an image but not saving it', async ({
   page
 }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByTestId('mini-avatar-container').click();
 
   await uploadAvatar(page);
@@ -151,7 +152,7 @@ test('it keeps the original pictogram on cancel after changing the pictogram but
   page
 }) => {
   const pictogramName = 'Analytics';
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByTestId('mini-avatar-container').click();
 
   await page.getByPlaceholder('Search').click();
@@ -171,7 +172,7 @@ test('it keeps the original pictogram on close (not cancel) after changing the p
   page
 }) => {
   const pictogramName = 'Analytics';
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByTestId('mini-avatar-container').click();
 
   await page.getByPlaceholder('Search').click();
@@ -191,7 +192,7 @@ test('it saves the pictogram if the save button is clicked on the pictogram tab
 }) => {
   const pictogramName = 'Analytics';
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByTestId('mini-avatar-container').click();
 
   await uploadAvatar(page);
@@ -212,7 +213,7 @@ test('it can upload an image, then change to a pictogram, then change to an imag
   const assistantInput = getFakeAssistantInput();
   const pictogramName = getRandomPictogramName();
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await fillOutRequiredAssistantFields(assistantInput, page);
 
@@ -232,7 +233,7 @@ test('it can upload an image, then change to a pictogram, then change to an imag
 
 test('it deletes the avatar image from storage when the avatar', async ({ page, openAIClient }) => {
   const assistant = await createAssistantWithApi({ openAIClient });
-  await page.goto(`/chat/assistants-management/edit/${assistant.id}`);
+  await loadEditAssistantPage(assistant.id, page);
   await saveAvatarImage(page);
   const saveButton = page.getByRole('button', { name: 'Save' }).nth(0);
   await saveButton.click();
@@ -246,7 +247,7 @@ test('it deletes the avatar image from storage when the avatar', async ({ page,
   const contentType = res.headers.get('content-type');
   expect(contentType).toMatch(/^image\//);
 
-  await page.goto(`/chat/assistants-management/edit/${assistant.id}`);
+  await loadEditAssistantPage(assistant.id, page);
   await savePictogram(getRandomPictogramName(), page);
   await saveButton.click();
   await expect(page.getByText('Assistant Updated')).toBeVisible();

src/leapfrogai_ui/tests/assistant-progress.test.ts

@@ -8,6 +8,7 @@ import {
   deleteFixtureFile,
   uploadFileWithApi
 } from './helpers/fileHelpers';
+import { loadNewAssistantPage } from './helpers/navigationHelpers';
 
 // Note - fully testing the assistant progress toast has proven difficult with Playwright. Sometimes the websocket
 // connection for the Supabase realtime listeners works, and sometimes it does not. Here we test that the
@@ -24,7 +25,7 @@ test('when creating an assistant with files, an assistant progress toast is disp
   const uploadedFile1 = await uploadFileWithApi(filename1, 'application/pdf', openAIClient);
   const uploadedFile2 = await uploadFileWithApi(filename2, 'application/pdf', openAIClient);
 
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
 
   await page.getByLabel('name').fill(assistantInput.name);
   await page.getByLabel('tagline').fill(assistantInput.description);

src/leapfrogai_ui/tests/assistants.test.ts

@@ -11,7 +11,11 @@ import {
   getAssistantWithApi
 } from './helpers/assistantHelpers';
 import { deleteActiveThread, getLastUrlParam, sendMessage } from './helpers/threadHelpers';
-import { loadChatPage } from './helpers/navigationHelpers';
+import {
+  loadAssistantsManagementPage,
+  loadChatPage,
+  loadNewAssistantPage
+} from './helpers/navigationHelpers';
 
 test('it navigates to the assistants page', async ({ page }) => {
   await loadChatPage(page);
@@ -23,8 +27,7 @@ test('it navigates to the assistants page', async ({ page }) => {
 });
 
 test('it has a button that navigates to the new assistant page', async ({ page }) => {
-  await page.goto('/chat/assistants-management');
-
+  await loadAssistantsManagementPage(page);
   await page.getByRole('button', { name: 'New Assistant' }).click();
   await page.waitForURL('**/assistants-management/new');
   await expect(page).toHaveTitle('LeapfrogAI - New Assistant');
@@ -38,7 +41,7 @@ test('it creates an assistant and navigates back to the management page', async
 
   await createAssistant(assistantInput, page);
 
-  await expect(page.getByText('Assistant Created')).toBeVisible();
+  await expect(page.getByText('Assistant Created')).toBeVisible({ timeout: 10000 });
   await page.waitForURL('/chat/assistants-management');
   await expect(page.getByTestId(`assistant-card-${assistantInput.name}`)).toBeVisible();
 
@@ -86,7 +89,7 @@ test('displays an error toast when there is an error editing an assistant and re
   const assistant = await createAssistantWithApi({ openAIClient });
   const newAssistantAttributes = getFakeAssistantInput();
 
-  await page.goto('/chat/assistants-management');
+  await loadAssistantsManagementPage(page);
 
   await editAssistantCard(assistant.name!, page);
 
@@ -126,21 +129,19 @@ test('it can search for assistants', async ({ page, openAIClient }) => {
   const assistant1 = await createAssistantWithApi({ openAIClient });
   const assistant2 = await createAssistantWithApi({ openAIClient });
 
-  await page.goto('/chat/assistants-management');
+  await loadAssistantsManagementPage(page);
+
+  const searchBox = page.getByRole('textbox', {
+    name: /search/i
+  });
+  await expect(searchBox).toBeVisible();
   // Search by name
-  await page
-    .getByRole('textbox', {
-      name: /search/i
-    })
-    .fill(assistant1.name!);
+  await searchBox.fill(assistant1.name!);
 
   await expect(page.getByTestId(`assistant-card-${assistant2.name}`)).not.toBeVisible();
   await expect(page.getByTestId(`assistant-card-${assistant1.name}`)).toBeVisible();
 
   // search by description
-  const searchBox = page.getByRole('textbox', {
-    name: /search/i
-  });
   await searchBox.clear();
   await searchBox.fill(assistant2.description!);
 
@@ -188,7 +189,7 @@ test('it can navigate to the last visited thread with breadcrumbs', async ({
 });
 
 test('it validates input', async ({ page }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByLabel('name').fill('my assistant');
   const saveButton = page.getByRole('button', { name: 'Save' });
 
@@ -206,7 +207,7 @@ test('it validates input', async ({ page }) => {
 });
 
 test('it confirms you want to navigate away if you have changes', async ({ page }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByLabel('name').fill('my assistant');
 
   await page.getByRole('link', { name: 'Assistants Management' }).click();
@@ -220,8 +221,7 @@ test('it confirms you want to navigate away if you have changes', async ({ page
 });
 
 test('it DOES NOT confirm you want to navigate away if you DONT have changes', async ({ page }) => {
-  await page.goto('/chat/assistants-management/new');
-
+  await loadNewAssistantPage(page);
   await page.getByRole('link', { name: 'Assistants Management' }).click();
 
   await page.waitForURL('/chat/assistants-management');
@@ -230,7 +230,7 @@ test('it DOES NOT confirm you want to navigate away if you DONT have changes', a
 test('it DOES NOT confirm you want to navigate away if you click the cancel button', async ({
   page
 }) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByLabel('name').fill('my assistant');
 
   await page.getByRole('button', { name: 'Cancel' }).click();
@@ -243,7 +243,7 @@ test('it allows you to edit an assistant', async ({ page, openAIClient }) => {
   const assistant1 = await createAssistantWithApi({ openAIClient });
   const newAssistantAttributes = getFakeAssistantInput();
 
-  await page.goto('/chat/assistants-management');
+  await loadAssistantsManagementPage(page);
 
   await editAssistantCard(assistant1.name!, page);
 
@@ -274,7 +274,7 @@ test("it populates the assistants values when editing an assistant's details", a
 }) => {
   const assistant = await createAssistantWithApi({ openAIClient });
 
-  await page.goto('/chat/assistants-management');
+  await loadAssistantsManagementPage(page);
 
   await editAssistantCard(assistant.name!, page);
 
@@ -289,7 +289,7 @@ test("it populates the assistants values when editing an assistant's details", a
 test('it can delete assistants', async ({ page, openAIClient }) => {
   const assistant = await createAssistantWithApi({ openAIClient });
 
-  await page.goto('/chat/assistants-management');
+  await loadAssistantsManagementPage(page);
 
   await deleteAssistantCard(assistant.name!, page);
 

src/leapfrogai_ui/tests/chat.test.ts

@@ -176,34 +176,6 @@ test('it formats code in a code block and can copy the code', async ({ page }) =
   expect(copyBtns.length).toBeGreaterThan(0);
 });
 
-// The skeleton only shows for assistant messages
-// TODO -this test can be flaky if the backend is really fast and the loading-msg skeleton barely has time to be shown
-test.skip('it shows a loading skeleton when a response is pending', async ({
-  page,
-  openAIClient
-}) => {
-  const assistant = await createAssistantWithApi({ openAIClient });
-
-  await loadChatPage(page);
-
-  // Select assistant
-  await expect(page.getByTestId('assistants-select-btn')).not.toBeDisabled();
-  const assistantDropdown = page.getByTestId('assistants-select-btn');
-  await assistantDropdown.click();
-  await page.getByText(assistant!.name!).click();
-
-  const messages = page.getByTestId('message');
-  await sendMessage(page, newMessage1);
-  await expect(page.getByTestId('loading-msg')).toBeVisible();
-  await waitForResponseToComplete(page);
-  await expect(messages).toHaveCount(2);
-  await expect(page.getByTestId('loading-msg')).not.toBeVisible();
-
-  // Cleanup
-  await deleteActiveThread(page, openAIClient);
-  await deleteAssistantWithApi(assistant.id, openAIClient);
-});
-
 test('it can chat with an assistant that doesnt have files', async ({ page, openAIClient }) => {
   const assistant = await createAssistantWithApi({ openAIClient });
   expect(assistant.tool_resources?.file_search).not.toBeDefined(); // ensure the assistant has no files

src/leapfrogai_ui/tests/fileChatActions.test.ts

@@ -56,7 +56,7 @@ test('it can transcribe an audio file', async ({ page, openAIClient }) => {
   await deleteActiveThread(page, openAIClient);
 });
 
-test('it can removes the audio file but keeps other files after translating', async ({
+test('it can remove the audio file but keeps other files after translating', async ({
   page,
   openAIClient
 }) => {

src/leapfrogai_ui/tests/global.teardown.ts

@@ -1,10 +1,24 @@
 import { test } from '@playwright/test';
 import { cleanup } from './helpers/cleanup';
 import { getOpenAIClient } from './fixtures';
+import fs from 'node:fs';
 
 test('teardown', async () => {
   const openAIClient = await getOpenAIClient();
   console.log('cleaning up...');
   await cleanup(openAIClient);
+  // Check if the auth file exists and delete it
+  const filePath = 'playwright/.auth/user.json';
+  if (fs.existsSync(filePath)) {
+    fs.unlink(filePath, (err) => {
+      if (err) {
+        console.error('Error deleting the auth file:', err);
+      } else {
+        console.log('Auth file deleted successfully.');
+      }
+    });
+  } else {
+    console.log('Auth file not found.');
+  }
   console.log('clean up complete');
 });

src/leapfrogai_ui/tests/helpers/assistantHelpers.ts

@@ -5,9 +5,10 @@ import type { Assistant, AssistantCreateParams } from 'openai/resources/beta/ass
 import type { AssistantInput, LFAssistant } from '../../src/lib/types/assistants';
 import { supabase } from './helpers';
 import { faker } from '@faker-js/faker';
+import { loadNewAssistantPage } from './navigationHelpers';
 
 export const createAssistant = async (assistantInput: AssistantInput, page: Page) => {
-  await page.goto('/chat/assistants-management/new');
+  await loadNewAssistantPage(page);
   await page.getByLabel('name').fill(assistantInput.name);
   await page.getByLabel('tagline').fill(assistantInput.description);
   await clickOnSliderValue(assistantInput.temperature, page);
@@ -151,7 +152,9 @@ export const fillOutRequiredAssistantFields = async (
   assistantInput: AssistantInput,
   page: Page
 ) => {
-  await page.getByLabel('name').fill(assistantInput.name);
+  const nameField = page.getByLabel('name'); // ensure name field is visible before starting to fill out the form
+  await expect(nameField).toBeVisible();
+  await nameField.fill(assistantInput.name);
   await page.getByLabel('tagline').fill(assistantInput.description);
   await page.getByPlaceholder("You'll act as...").fill(assistantInput.instructions);
 };
@@ -161,7 +164,7 @@ export const saveAssistant = async (assistantName: string, page: Page) => {
   await expect(saveButtons).toHaveCount(1);
   await saveButtons.click();
 
-  await expect(page.getByText('Assistant Created')).toBeVisible();
+  await expect(page.getByText('Assistant Created')).toBeVisible({ timeout: 10000 });
   await page.waitForURL('/chat/assistants-management');
   await expect(page.getByTestId(`assistant-card-${assistantName}`)).toBeVisible();
 };