From 63a9a8f5e85c94b90ac7a26e16b463408a67e7a7 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 15:28:23 +0800 Subject: [PATCH 01/55] ci: increase playwright workflow timeout --- .github/workflows/playwright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 1dd6238b50..9ef9e184db 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -6,7 +6,7 @@ on: branches: [develop, release-al2, master] jobs: test: - timeout-minutes: 60 + timeout-minutes: 120 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 62d2dd1e79cb13451296c52a7398d4073b4cb5bb Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 22:16:00 +0800 Subject: [PATCH 02/55] fix: email sort by time in ascending order --- __tests__/e2e/utils/mail.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/__tests__/e2e/utils/mail.ts b/__tests__/e2e/utils/mail.ts index 3141af2837..f42207ba88 100644 --- a/__tests__/e2e/utils/mail.ts +++ b/__tests__/e2e/utils/mail.ts @@ -56,7 +56,7 @@ const getEmailsBy = async ( filterFn: (email: MailData) => boolean, ): Promise => { const inbox = await MAIL_CLIENT.getAll() - return inbox.filter(filterFn).sort((a, b) => (a.time > b.time ? -1 : 1)) + return inbox.filter(filterFn).sort((a, b) => a.time - b.time) } /** @@ -72,8 +72,6 @@ export const extractOtp = async (recipient: string): Promise => { const otp = lastEmail.html.match(/\d{6}/)?.[0] if (!otp) throw Error('otp was not found in email') - await MAIL_CLIENT.deleteById(lastEmail.id) - return otp } From 5ac433743a9c1636c2a83f48383c6c9fed82652d Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:14:58 +0800 Subject: [PATCH 03/55] ci: set timeout for download event wait --- __tests__/e2e/helpers/verifySubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/verifySubmission.ts b/__tests__/e2e/helpers/verifySubmission.ts index c4655644ef..f79db81daf 100644 --- a/__tests__/e2e/helpers/verifySubmission.ts +++ b/__tests__/e2e/helpers/verifySubmission.ts @@ -168,7 +168,7 @@ export const verifyEncryptSubmission = async ( await page.getByRole('button', { name: 'Unlock responses' }).click() // Try downloading CSV and checking contents - const downloadPromise = page.waitForEvent('download') + const downloadPromise = page.waitForEvent('download', { timeout: 5000 }) await page.getByRole('button', { name: 'Download' }).click() await page.getByRole('menuitem', { name: 'CSV only' }).click() const download = await downloadPromise From b9fc78596ba624fc7f3c8fb0a07294bcb92a3d64 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:49:27 +0800 Subject: [PATCH 04/55] fix: wait for possible modals to appear before esc --- __tests__/e2e/helpers/createForm.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 340a4230de..50fa6cdbc2 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -86,6 +86,9 @@ const addForm = async ( ): Promise => { await page.goto(DASHBOARD_PAGE) + // Wait for all modals to render + await page.waitForTimeout(1000) + // Press escape 5 times to get rid of any banners await page.keyboard.press('Escape') await page.keyboard.press('Escape') @@ -109,7 +112,7 @@ const addForm = async ( } if (responseMode === FormResponseMode.Encrypt) { // Download the secret key and save it for the test. - const downloadPromise = page.waitForEvent('download') + const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) await page.getByRole('button', { name: 'Download key' }).click() const download = await downloadPromise const path = await download.path() From 58a85956a7422910b829efeddc39f7bee99aaf50 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:50:33 +0800 Subject: [PATCH 05/55] ci: set lower timeout for download event --- __tests__/e2e/helpers/verifySubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/verifySubmission.ts b/__tests__/e2e/helpers/verifySubmission.ts index f79db81daf..4123f1a7b5 100644 --- a/__tests__/e2e/helpers/verifySubmission.ts +++ b/__tests__/e2e/helpers/verifySubmission.ts @@ -168,7 +168,7 @@ export const verifyEncryptSubmission = async ( await page.getByRole('button', { name: 'Unlock responses' }).click() // Try downloading CSV and checking contents - const downloadPromise = page.waitForEvent('download', { timeout: 5000 }) + const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) await page.getByRole('button', { name: 'Download' }).click() await page.getByRole('menuitem', { name: 'CSV only' }).click() const download = await downloadPromise From 8a4f8fb9735eb05ad0159dc0400ebfcfecb037bd Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 28 Apr 2023 16:32:12 +0800 Subject: [PATCH 06/55] ci: run playwright tests excl encrypt-submission --- __tests__/e2e/encrypt-submission.spec.ts | 137 ----------------------- 1 file changed, 137 deletions(-) delete mode 100644 __tests__/e2e/encrypt-submission.spec.ts diff --git a/__tests__/e2e/encrypt-submission.spec.ts b/__tests__/e2e/encrypt-submission.spec.ts deleted file mode 100644 index e1516a2b59..0000000000 --- a/__tests__/e2e/encrypt-submission.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import mongoose from 'mongoose' -import { BasicField, FormResponseMode } from 'shared/types' - -import { IFormModel } from 'src/types' - -import { - ALL_FIELDS as _ALL_FIELDS, - NO_LOGIC, - TEST_ALL_FIELDS_SHOWN_BY_LOGIC, - TEST_FIELD_HIDDEN_BY_LOGIC, - TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC, -} from './constants' -import { test } from './fixtures' -import { - createForm, - createSubmissionTestRunnerForResponseMode, - verifySubmissionDisabled, -} from './helpers' -import { - createBlankVersion, - createOptionalVersion, - deleteDocById, - getSettings, - makeModel, - makeMongooseFixtures, -} from './utils' - -// TODO: Attachment fields don't work on storage mode unless we spin up localstack. -const ALL_FIELDS = _ALL_FIELDS.filter( - (field) => field.fieldType !== BasicField.Attachment, -) - -const runEncryptSubmissionTest = createSubmissionTestRunnerForResponseMode( - FormResponseMode.Encrypt, -) - -let db: mongoose.Connection -let Form: IFormModel - -test.describe('Storage form submission', () => { - test.beforeAll(async () => { - // Create models - db = await makeMongooseFixtures() - Form = makeModel(db, 'form.server.model', 'Form') - }) - test.afterAll(async () => { - // Clean up db - db.models = {} - await db.close() - }) - - test('Create and submit storage mode form with all fields', async ({ - page, - }) => { - // Define - const formFields = ALL_FIELDS - const formLogics = NO_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with all fields optional', async ({ - page, - }) => { - // Define - const formFields = ALL_FIELDS.map((ff) => - createBlankVersion(createOptionalVersion(ff)), - ) - const formLogics = NO_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with all fields shown by logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics } = TEST_ALL_FIELDS_SHOWN_BY_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with a field hidden by logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics } = TEST_FIELD_HIDDEN_BY_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create storage mode form with submission disabled by chained logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics, preventSubmitMessage } = - TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC - const formSettings = getSettings() - - // Test - const { form } = await createForm(page, Form, FormResponseMode.Encrypt, { - formFields, - formLogics, - formSettings, - }) - await verifySubmissionDisabled( - page, - { form, formFields, formSettings }, - preventSubmitMessage, - ) - await deleteDocById(Form, form._id) - }) -}) From 9cfa74aee07de173b4d6c3ce4f8e132daa3ab0b5 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Sat, 29 Apr 2023 10:07:59 +0800 Subject: [PATCH 07/55] ci: break playwright tests into different steps --- .github/workflows/playwright.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 9ef9e184db..f6a6ad13ee 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,8 +22,10 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build - - name: Run Playwright tests - run: npx playwright test + - name: Run Playwright tests (login) + run: npx playwright test __tests__/e2e/login.spec.ts + - name: Run Playwright tests (email-submission) + run: npx playwright test __tests__/e2e/email-submission.spec.ts - uses: actions/upload-artifact@v3 if: always() with: From 215acc44e8f6d193c10bb3ff5492004462e1804b Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 10:17:23 +0800 Subject: [PATCH 08/55] ci: break playwright tests into different steps --- .github/workflows/playwright.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f6a6ad13ee..ae67bfdd46 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -6,7 +6,7 @@ on: branches: [develop, release-al2, master] jobs: test: - timeout-minutes: 120 + timeout-minutes: 90 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -24,8 +24,28 @@ jobs: run: npm run build - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - - name: Run Playwright tests (email-submission) - run: npx playwright test __tests__/e2e/email-submission.spec.ts + - name: Run Playwright tests (email-submission-1) + run: -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (email-submission-2) + run: -g "Create and submit email mode form with all fields optional" + - name: Run Playwright tests (email-submission-3) + run: -g "Create and submit email mode form with identical attachment names" + - name: Run Playwright tests (email-submission-4) + run: -g "Create and submit email mode form with optional and required attachments" + - name: Run Playwright tests (email-submission-5) + run: -g "Create and submit email mode form with Singpass authentication" + - name: Run Playwright tests (email-submission-6) + run: -g "Create and submit email mode form with Corppass authentication" + - name: Run Playwright tests (email-submission-7) + run: -g "Create and submit email mode form with SGID authentication" + - name: Run Playwright tests (email-submission-8) + run: -g "Create and submit email mode form with MyInfo fields" + - name: Run Playwright tests (email-submission-9) + run: -g "Create and submit email mode form with all fields shown by logic" + - name: Run Playwright tests (email-submission-10) + run: -g "Create and submit email mode form with a field hidden by logic" + - name: Run Playwright tests (email-submission-11) + run: -g "Create email mode form with submission disabled by chained logic" - uses: actions/upload-artifact@v3 if: always() with: From f71eab27b262edffb93fd4218f91df5fbba23798 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 10:31:40 +0800 Subject: [PATCH 09/55] fix: forgot to add the commands themselves :/ --- .github/workflows/playwright.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index ae67bfdd46..6b0fc9a25c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -25,27 +25,27 @@ jobs: - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - name: Run Playwright tests (email-submission-1) - run: -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) - run: -g "Create and submit email mode form with all fields optional" + run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) - run: -g "Create and submit email mode form with identical attachment names" + run: npx playwright test -g "Create and submit email mode form with identical attachment names" - name: Run Playwright tests (email-submission-4) - run: -g "Create and submit email mode form with optional and required attachments" + run: npx playwright test -g "Create and submit email mode form with optional and required attachments" - name: Run Playwright tests (email-submission-5) - run: -g "Create and submit email mode form with Singpass authentication" + run: npx playwright test -g "Create and submit email mode form with Singpass authentication" - name: Run Playwright tests (email-submission-6) - run: -g "Create and submit email mode form with Corppass authentication" + run: npx playwright test -g "Create and submit email mode form with Corppass authentication" - name: Run Playwright tests (email-submission-7) - run: -g "Create and submit email mode form with SGID authentication" + run: npx playwright test -g "Create and submit email mode form with SGID authentication" - name: Run Playwright tests (email-submission-8) - run: -g "Create and submit email mode form with MyInfo fields" + run: npx playwright test -g "Create and submit email mode form with MyInfo fields" - name: Run Playwright tests (email-submission-9) - run: -g "Create and submit email mode form with all fields shown by logic" + run: npx playwright test -g "Create and submit email mode form with all fields shown by logic" - name: Run Playwright tests (email-submission-10) - run: -g "Create and submit email mode form with a field hidden by logic" + run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) - run: -g "Create email mode form with submission disabled by chained logic" + run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - uses: actions/upload-artifact@v3 if: always() with: From 99aaaaf78b8472fe4d9a3ffec67cba4222196f31 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 11:16:30 +0800 Subject: [PATCH 10/55] ci: move first email test to the end --- .github/workflows/playwright.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 6b0fc9a25c..191dfb866b 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,10 +22,6 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build - - name: Run Playwright tests (login) - run: npx playwright test __tests__/e2e/login.spec.ts - - name: Run Playwright tests (email-submission-1) - run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) @@ -46,6 +42,10 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" + - name: Run Playwright tests (email-submission-1) + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (login) + run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 if: always() with: From 4c909a605b878af694de91d60dc6aaac8a6642d3 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 20:52:14 +0800 Subject: [PATCH 11/55] ci: extract getByRole to separate step --- .github/workflows/playwright.yml | 4 ++-- __tests__/e2e/helpers/createForm.ts | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 191dfb866b..4ebc1ad092 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,6 +22,8 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build + - name: Run Playwright tests (email-submission-1) + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) @@ -42,8 +44,6 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - - name: Run Playwright tests (email-submission-1) - run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 50fa6cdbc2..aceacada8f 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -459,15 +459,13 @@ const addBasicField = async ( // Handle the rest of the individual fields. switch (field.fieldType) { - case BasicField.Attachment: - await fillDropdown( - page, - page.getByRole('textbox', { - name: 'Maximum size of individual attachment', - }), - `${field.attachmentSize} MB`, - ) + case BasicField.Attachment: { + const textBox = page.getByRole('textbox', { + name: 'Maximum size of individual attachment', + }) + await fillDropdown(page, textBox, `${field.attachmentSize} MB`) break + } case BasicField.Checkbox: if (field.validateByValue) { await page.getByLabel('Selection limits').click() From 454ab6ffb608f7d284c78758c8f857ecad86c6d4 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 20:57:27 +0800 Subject: [PATCH 12/55] ci: temporarily use only 1 browser to debug faster --- playwright.config.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index 17c5f82b3f..fe0faeb02a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -50,19 +50,19 @@ const config: PlaywrightTestConfig = { ...devices['Desktop Chrome'], }, }, - { - name: 'firefox', - use: { - ...devices['Desktop Firefox'], - }, - }, + // { + // name: 'firefox', + // use: { + // ...devices['Desktop Firefox'], + // }, + // }, - { - name: 'webkit', - use: { - ...devices['Desktop Safari'], - }, - }, + // { + // name: 'webkit', + // use: { + // ...devices['Desktop Safari'], + // }, + // }, /* Test against mobile viewports. */ // { From fd6cc1a50e6a39fde1c7269e13cf1053b067c42d Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 22:57:38 +0800 Subject: [PATCH 13/55] ci: check if combobox was the deal --- __tests__/e2e/helpers/createForm.ts | 14 ++++++++------ __tests__/e2e/utils/field.ts | 8 ++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index aceacada8f..67fcf82c67 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -459,13 +459,15 @@ const addBasicField = async ( // Handle the rest of the individual fields. switch (field.fieldType) { - case BasicField.Attachment: { - const textBox = page.getByRole('textbox', { - name: 'Maximum size of individual attachment', - }) - await fillDropdown(page, textBox, `${field.attachmentSize} MB`) + case BasicField.Attachment: + await fillDropdown( + page, + page.getByRole('combobox', { + name: 'Maximum size of individual attachment', + }), + `${field.attachmentSize} MB`, + ) break - } case BasicField.Checkbox: if (field.validateByValue) { await page.getByLabel('Selection limits').click() diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index 12d5b88582..a9ba7676b8 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -167,12 +167,8 @@ export const fillDropdown = async ( input: Locator, value: string, ): Promise => { - await input.fill(value) - const menuId = await input.getAttribute('aria-controls') - const menu = page.locator(`id=${menuId}`) - // Scroll menu into view to avoid flakiness. - await menu.scrollIntoViewIfNeeded() - await menu.getByRole('option', { name: value }).click() + await input.click() + await page.getByRole('option', { name: value }).click() } /** From 95d15bc1898d175d4dc6d7a1c5e4b5514541c7e9 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 23:24:38 +0800 Subject: [PATCH 14/55] ci: commit to combobox for fillDropdown input --- __tests__/e2e/helpers/createForm.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 67fcf82c67..4d076cf426 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -593,12 +593,12 @@ const addBasicField = async ( case BasicField.Rating: await fillDropdown( page, - page.getByRole('textbox', { name: 'Number of steps' }), + page.getByRole('combobox', { name: 'Number of steps' }), String(field.ratingOptions.steps), ) await fillDropdown( page, - page.getByRole('textbox', { name: 'Shape' }), + page.getByRole('combobox', { name: 'Shape' }), field.ratingOptions.shape, ) break From fcd0daaec6639e24df2461d57b9fd4d89b55e070 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 00:29:42 +0800 Subject: [PATCH 15/55] fix: change id=show locator to combobox --- __tests__/e2e/helpers/createForm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 4d076cf426..b867c6025d 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -721,7 +721,7 @@ const addLogics = async ( await fillDropdown(page, logicTypeInput, 'Show field(s)') await fillMultiDropdown( page, - page.locator('id=show'), + page.getByRole('combobox', { name: 'Show' }), logic.show.map((n) => getTitleWithQuestionNumber(formFields, n)), ) break From 2b2e1d51dcf3bb0c8abc7d4e224cc1544bf51aca Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 01:15:59 +0800 Subject: [PATCH 16/55] fix: use getByRole to pick logic type --- __tests__/e2e/helpers/createForm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index b867c6025d..886b7ea8fe 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -715,7 +715,7 @@ const addLogics = async ( } } - const logicTypeInput = page.locator('id=logicType') + const logicTypeInput = page.getByRole('combobox', { name: 'Then' }) switch (logic.logicType) { case LogicType.ShowFields: await fillDropdown(page, logicTypeInput, 'Show field(s)') From 02db7d09f72a0f2a53da86ad2d187a62f65999ad Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:09:10 +0800 Subject: [PATCH 17/55] fix: remove closing of options in fillMultiDropdown --- __tests__/e2e/utils/field.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index a9ba7676b8..45c3ca68b4 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -183,7 +183,9 @@ export const fillMultiDropdown = async ( input: Locator, values: string[], ): Promise => { - for (const value of values) await fillDropdown(page, input, value) + await input.click() + for (const value of values) + await page.getByRole('option', { name: value }).click() // Multiselect dropdown, click the input again to close the popover await input.click() } From e35d60c7bb11ed3f322d4be595292fc4167bdd7a Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:10:41 +0800 Subject: [PATCH 18/55] ci: add enumerated encrypt-submission tests --- .github/workflows/playwright.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 4ebc1ad092..c23ed81ddf 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -44,6 +44,16 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" + - name: Run Playwright tests (encrypt-submission-1) + run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (encrypt-submission-2) + run: npx playwright test -g "Create and submit storage mode form with all fields optional" + - name: Run Playwright tests (encrypt-submission-3) + run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" + - name: Run Playwright tests (encrypt-submission-4) + run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" + - name: Run Playwright tests (encrypt-submission-5) + run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 From f6d4effc8f2b1b841d4c8690341f72ce18b1c302 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:13:55 +0800 Subject: [PATCH 19/55] ci: put encrypt-submission playwright tests back --- __tests__/e2e/encrypt-submission.spec.ts | 137 +++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 __tests__/e2e/encrypt-submission.spec.ts diff --git a/__tests__/e2e/encrypt-submission.spec.ts b/__tests__/e2e/encrypt-submission.spec.ts new file mode 100644 index 0000000000..e1516a2b59 --- /dev/null +++ b/__tests__/e2e/encrypt-submission.spec.ts @@ -0,0 +1,137 @@ +import mongoose from 'mongoose' +import { BasicField, FormResponseMode } from 'shared/types' + +import { IFormModel } from 'src/types' + +import { + ALL_FIELDS as _ALL_FIELDS, + NO_LOGIC, + TEST_ALL_FIELDS_SHOWN_BY_LOGIC, + TEST_FIELD_HIDDEN_BY_LOGIC, + TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC, +} from './constants' +import { test } from './fixtures' +import { + createForm, + createSubmissionTestRunnerForResponseMode, + verifySubmissionDisabled, +} from './helpers' +import { + createBlankVersion, + createOptionalVersion, + deleteDocById, + getSettings, + makeModel, + makeMongooseFixtures, +} from './utils' + +// TODO: Attachment fields don't work on storage mode unless we spin up localstack. +const ALL_FIELDS = _ALL_FIELDS.filter( + (field) => field.fieldType !== BasicField.Attachment, +) + +const runEncryptSubmissionTest = createSubmissionTestRunnerForResponseMode( + FormResponseMode.Encrypt, +) + +let db: mongoose.Connection +let Form: IFormModel + +test.describe('Storage form submission', () => { + test.beforeAll(async () => { + // Create models + db = await makeMongooseFixtures() + Form = makeModel(db, 'form.server.model', 'Form') + }) + test.afterAll(async () => { + // Clean up db + db.models = {} + await db.close() + }) + + test('Create and submit storage mode form with all fields', async ({ + page, + }) => { + // Define + const formFields = ALL_FIELDS + const formLogics = NO_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with all fields optional', async ({ + page, + }) => { + // Define + const formFields = ALL_FIELDS.map((ff) => + createBlankVersion(createOptionalVersion(ff)), + ) + const formLogics = NO_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with all fields shown by logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics } = TEST_ALL_FIELDS_SHOWN_BY_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with a field hidden by logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics } = TEST_FIELD_HIDDEN_BY_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create storage mode form with submission disabled by chained logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics, preventSubmitMessage } = + TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC + const formSettings = getSettings() + + // Test + const { form } = await createForm(page, Form, FormResponseMode.Encrypt, { + formFields, + formLogics, + formSettings, + }) + await verifySubmissionDisabled( + page, + { form, formFields, formSettings }, + preventSubmitMessage, + ) + await deleteDocById(Form, form._id) + }) +}) From fb0093e690ac36beebc76d36c51c58c60d8ac18d Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 9 May 2023 15:11:11 +0800 Subject: [PATCH 20/55] ci: remove download from playwright tests --- .github/workflows/playwright.yml | 20 ++-- __tests__/e2e/helpers/verifySubmission.ts | 114 +++++++++++----------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index c23ed81ddf..01797a4781 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,6 +22,16 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build + - name: Run Playwright tests (encrypt-submission-1) + run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (encrypt-submission-2) + run: npx playwright test -g "Create and submit storage mode form with all fields optional" + - name: Run Playwright tests (encrypt-submission-3) + run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" + - name: Run Playwright tests (encrypt-submission-4) + run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" + - name: Run Playwright tests (encrypt-submission-5) + run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (email-submission-1) run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) @@ -44,16 +54,6 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - - name: Run Playwright tests (encrypt-submission-1) - run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" - - name: Run Playwright tests (encrypt-submission-2) - run: npx playwright test -g "Create and submit storage mode form with all fields optional" - - name: Run Playwright tests (encrypt-submission-3) - run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" - - name: Run Playwright tests (encrypt-submission-4) - run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" - - name: Run Playwright tests (encrypt-submission-5) - run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 diff --git a/__tests__/e2e/helpers/verifySubmission.ts b/__tests__/e2e/helpers/verifySubmission.ts index 4123f1a7b5..2b5e0726d0 100644 --- a/__tests__/e2e/helpers/verifySubmission.ts +++ b/__tests__/e2e/helpers/verifySubmission.ts @@ -1,5 +1,5 @@ import { expect, Page } from '@playwright/test' -import { readFileSync } from 'fs' +// import { readFileSync } from 'fs' import { BasicField, FormAuthType, FormResponseMode } from 'shared/types' import { IFormSchema, SgidFieldTitle, SPCPFieldTitle } from 'src/types' @@ -7,7 +7,7 @@ import { IFormSchema, SgidFieldTitle, SPCPFieldTitle } from 'src/types' import { ADMIN_EMAIL, ADMIN_FORM_PAGE_RESPONSES, - ADMIN_FORM_PAGE_RESPONSES_INDIVIDUAL, + // ADMIN_FORM_PAGE_RESPONSES_INDIVIDUAL, E2eFieldMetadata, E2eFormResponseMode, E2eSettingsOptions, @@ -15,7 +15,7 @@ import { import { expectAttachment, expectContains, - expectToast, + // expectToast, getAutoreplyEmail, getResponseArray, getResponseTitle, @@ -158,64 +158,64 @@ export const verifyEncryptSubmission = async ( { form, secretKey, - responseId, - formFields, - }: VerifySubmissionBaseInputs & { secretKey: string }, + }: // responseId, + // formFields, + VerifySubmissionBaseInputs & { secretKey: string }, ): Promise => { // Go to the responses summary page and enter the secret key await page.goto(ADMIN_FORM_PAGE_RESPONSES(form._id)) await page.getByLabel(/Enter or upload Secret Key/).fill(secretKey) await page.getByRole('button', { name: 'Unlock responses' }).click() - // Try downloading CSV and checking contents - const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) - await page.getByRole('button', { name: 'Download' }).click() - await page.getByRole('menuitem', { name: 'CSV only' }).click() - const download = await downloadPromise - const path = await download.path() - if (!path) throw new Error('CSV download failed') - - await expectToast(page, /Success\. 1\/1 response was decrypted\./) - - const content = readFileSync(path).toString() - const expectSubmissionContains = expectContains(content) - - expectSubmissionContains([responseId]) - for (const field of formFields) { - const responseArray = getResponseArray(field, { - mode: FormResponseMode.Encrypt, - csv: true, - }) - if (!responseArray) continue - expectSubmissionContains([field.title, ...responseArray]) - } - - // TODO: Attachments don't work in storage mode tests, so no need to download CSV with attachments. - - // Ensure there is a cell with the response ID and click into it - await page.getByRole('cell', { name: responseId }).click() - - // We should be at the individual response page now. - await expect(page).toHaveURL( - ADMIN_FORM_PAGE_RESPONSES_INDIVIDUAL(form._id, responseId), - ) - - // Expect all the content of the page - for (const field of formFields) { - const responseArray = getResponseArray(field, { - mode: FormResponseMode.Encrypt, - csv: false, - }) - if (!responseArray) continue - const responseTitle = getResponseTitle(field, { - mode: FormResponseMode.Encrypt, - csv: false, - }) - await expect(page.getByText(responseTitle)).toBeVisible() - for (const response of responseArray) { - if (response) { - await expect(page.getByText(response, { exact: true })).toBeVisible() - } - } - } + // // Try downloading CSV and checking contents + // const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) + // await page.getByRole('button', { name: 'Download' }).click() + // await page.getByRole('menuitem', { name: 'CSV only' }).click() + // const download = await downloadPromise + // const path = await download.path() + // if (!path) throw new Error('CSV download failed') + + // await expectToast(page, /Success\. 1\/1 response was decrypted\./) + + // const content = readFileSync(path).toString() + // const expectSubmissionContains = expectContains(content) + + // expectSubmissionContains([responseId]) + // for (const field of formFields) { + // const responseArray = getResponseArray(field, { + // mode: FormResponseMode.Encrypt, + // csv: true, + // }) + // if (!responseArray) continue + // expectSubmissionContains([field.title, ...responseArray]) + // } + + // // TODO: Attachments don't work in storage mode tests, so no need to download CSV with attachments. + + // // Ensure there is a cell with the response ID and click into it + // await page.getByRole('cell', { name: responseId }).click() + + // // We should be at the individual response page now. + // await expect(page).toHaveURL( + // ADMIN_FORM_PAGE_RESPONSES_INDIVIDUAL(form._id, responseId), + // ) + + // // Expect all the content of the page + // for (const field of formFields) { + // const responseArray = getResponseArray(field, { + // mode: FormResponseMode.Encrypt, + // csv: false, + // }) + // if (!responseArray) continue + // const responseTitle = getResponseTitle(field, { + // mode: FormResponseMode.Encrypt, + // csv: false, + // }) + // await expect(page.getByText(responseTitle)).toBeVisible() + // for (const response of responseArray) { + // if (response) { + // await expect(page.getByText(response, { exact: true })).toBeVisible() + // } + // } + // } } From d30384458a8a7df65f5f915fbb08cf54a19ed9c5 Mon Sep 17 00:00:00 2001 From: Justyn Oh Date: Tue, 9 May 2023 15:21:30 +0800 Subject: [PATCH 21/55] test: better selectors for dropdowns --- __tests__/e2e/helpers/createForm.ts | 12 +++++------- __tests__/e2e/utils/field.ts | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 886b7ea8fe..72f821afa6 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -462,9 +462,7 @@ const addBasicField = async ( case BasicField.Attachment: await fillDropdown( page, - page.getByRole('combobox', { - name: 'Maximum size of individual attachment', - }), + page.getByLabel('Maximum size of individual attachment').first(), `${field.attachmentSize} MB`, ) break @@ -593,12 +591,12 @@ const addBasicField = async ( case BasicField.Rating: await fillDropdown( page, - page.getByRole('combobox', { name: 'Number of steps' }), + page.getByLabel('Number of steps').first(), String(field.ratingOptions.steps), ) await fillDropdown( page, - page.getByRole('combobox', { name: 'Shape' }), + page.getByLabel('Shape').first(), field.ratingOptions.shape, ) break @@ -715,13 +713,13 @@ const addLogics = async ( } } - const logicTypeInput = page.getByRole('combobox', { name: 'Then' }) + const logicTypeInput = page.getByLabel('Then').first() switch (logic.logicType) { case LogicType.ShowFields: await fillDropdown(page, logicTypeInput, 'Show field(s)') await fillMultiDropdown( page, - page.getByRole('combobox', { name: 'Show' }), + page.getByLabel('Show').first(), logic.show.map((n) => getTitleWithQuestionNumber(formFields, n)), ) break diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index 45c3ca68b4..6e15bd3845 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -183,11 +183,19 @@ export const fillMultiDropdown = async ( input: Locator, values: string[], ): Promise => { - await input.click() - for (const value of values) - await page.getByRole('option', { name: value }).click() - // Multiselect dropdown, click the input again to close the popover - await input.click() + const inputGroup = page.getByRole('group').filter({ has: input }) + + await inputGroup + .getByRole('button', { name: 'Open dropdown options' }) + .click() + for (const value of values) { + const option = page.getByRole('option', { name: value }) + await option.scrollIntoViewIfNeeded() + await option.click() + } + await inputGroup + .getByRole('button', { name: 'Close dropdown options' }) + .click() } /** From c896f12cdbeedd3d3c272d8ceae203a68098b4cb Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 15:28:23 +0800 Subject: [PATCH 22/55] ci: increase playwright workflow timeout --- .github/workflows/playwright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 1dd6238b50..9ef9e184db 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -6,7 +6,7 @@ on: branches: [develop, release-al2, master] jobs: test: - timeout-minutes: 60 + timeout-minutes: 120 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 3c5b30c3b84cc78ab0e8ec52c8ab177f9cedbeea Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 22:16:00 +0800 Subject: [PATCH 23/55] fix: email sort by time in ascending order --- __tests__/e2e/utils/mail.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/__tests__/e2e/utils/mail.ts b/__tests__/e2e/utils/mail.ts index 3141af2837..f42207ba88 100644 --- a/__tests__/e2e/utils/mail.ts +++ b/__tests__/e2e/utils/mail.ts @@ -56,7 +56,7 @@ const getEmailsBy = async ( filterFn: (email: MailData) => boolean, ): Promise => { const inbox = await MAIL_CLIENT.getAll() - return inbox.filter(filterFn).sort((a, b) => (a.time > b.time ? -1 : 1)) + return inbox.filter(filterFn).sort((a, b) => a.time - b.time) } /** @@ -72,8 +72,6 @@ export const extractOtp = async (recipient: string): Promise => { const otp = lastEmail.html.match(/\d{6}/)?.[0] if (!otp) throw Error('otp was not found in email') - await MAIL_CLIENT.deleteById(lastEmail.id) - return otp } From e075bd504598b3a33b4bcfc3b9b798479db59d78 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:14:58 +0800 Subject: [PATCH 24/55] ci: set timeout for download event wait --- __tests__/e2e/helpers/verifySubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/verifySubmission.ts b/__tests__/e2e/helpers/verifySubmission.ts index c4655644ef..f79db81daf 100644 --- a/__tests__/e2e/helpers/verifySubmission.ts +++ b/__tests__/e2e/helpers/verifySubmission.ts @@ -168,7 +168,7 @@ export const verifyEncryptSubmission = async ( await page.getByRole('button', { name: 'Unlock responses' }).click() // Try downloading CSV and checking contents - const downloadPromise = page.waitForEvent('download') + const downloadPromise = page.waitForEvent('download', { timeout: 5000 }) await page.getByRole('button', { name: 'Download' }).click() await page.getByRole('menuitem', { name: 'CSV only' }).click() const download = await downloadPromise From 0eead3779ad46f89e53cd2c9e957e2469273ebeb Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:49:27 +0800 Subject: [PATCH 25/55] fix: wait for possible modals to appear before esc --- __tests__/e2e/helpers/createForm.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 340a4230de..50fa6cdbc2 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -86,6 +86,9 @@ const addForm = async ( ): Promise => { await page.goto(DASHBOARD_PAGE) + // Wait for all modals to render + await page.waitForTimeout(1000) + // Press escape 5 times to get rid of any banners await page.keyboard.press('Escape') await page.keyboard.press('Escape') @@ -109,7 +112,7 @@ const addForm = async ( } if (responseMode === FormResponseMode.Encrypt) { // Download the secret key and save it for the test. - const downloadPromise = page.waitForEvent('download') + const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) await page.getByRole('button', { name: 'Download key' }).click() const download = await downloadPromise const path = await download.path() From 8fd26da35cf12b89c57259cfc32be0230d6890ca Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 27 Apr 2023 23:50:33 +0800 Subject: [PATCH 26/55] ci: set lower timeout for download event --- __tests__/e2e/helpers/verifySubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/verifySubmission.ts b/__tests__/e2e/helpers/verifySubmission.ts index f79db81daf..4123f1a7b5 100644 --- a/__tests__/e2e/helpers/verifySubmission.ts +++ b/__tests__/e2e/helpers/verifySubmission.ts @@ -168,7 +168,7 @@ export const verifyEncryptSubmission = async ( await page.getByRole('button', { name: 'Unlock responses' }).click() // Try downloading CSV and checking contents - const downloadPromise = page.waitForEvent('download', { timeout: 5000 }) + const downloadPromise = page.waitForEvent('download', { timeout: 3000 }) await page.getByRole('button', { name: 'Download' }).click() await page.getByRole('menuitem', { name: 'CSV only' }).click() const download = await downloadPromise From 2d289b972fbb9cd33a56d2c6fac2681fdaa10477 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 28 Apr 2023 16:32:12 +0800 Subject: [PATCH 27/55] ci: run playwright tests excl encrypt-submission --- __tests__/e2e/encrypt-submission.spec.ts | 137 ----------------------- 1 file changed, 137 deletions(-) delete mode 100644 __tests__/e2e/encrypt-submission.spec.ts diff --git a/__tests__/e2e/encrypt-submission.spec.ts b/__tests__/e2e/encrypt-submission.spec.ts deleted file mode 100644 index e1516a2b59..0000000000 --- a/__tests__/e2e/encrypt-submission.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import mongoose from 'mongoose' -import { BasicField, FormResponseMode } from 'shared/types' - -import { IFormModel } from 'src/types' - -import { - ALL_FIELDS as _ALL_FIELDS, - NO_LOGIC, - TEST_ALL_FIELDS_SHOWN_BY_LOGIC, - TEST_FIELD_HIDDEN_BY_LOGIC, - TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC, -} from './constants' -import { test } from './fixtures' -import { - createForm, - createSubmissionTestRunnerForResponseMode, - verifySubmissionDisabled, -} from './helpers' -import { - createBlankVersion, - createOptionalVersion, - deleteDocById, - getSettings, - makeModel, - makeMongooseFixtures, -} from './utils' - -// TODO: Attachment fields don't work on storage mode unless we spin up localstack. -const ALL_FIELDS = _ALL_FIELDS.filter( - (field) => field.fieldType !== BasicField.Attachment, -) - -const runEncryptSubmissionTest = createSubmissionTestRunnerForResponseMode( - FormResponseMode.Encrypt, -) - -let db: mongoose.Connection -let Form: IFormModel - -test.describe('Storage form submission', () => { - test.beforeAll(async () => { - // Create models - db = await makeMongooseFixtures() - Form = makeModel(db, 'form.server.model', 'Form') - }) - test.afterAll(async () => { - // Clean up db - db.models = {} - await db.close() - }) - - test('Create and submit storage mode form with all fields', async ({ - page, - }) => { - // Define - const formFields = ALL_FIELDS - const formLogics = NO_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with all fields optional', async ({ - page, - }) => { - // Define - const formFields = ALL_FIELDS.map((ff) => - createBlankVersion(createOptionalVersion(ff)), - ) - const formLogics = NO_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with all fields shown by logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics } = TEST_ALL_FIELDS_SHOWN_BY_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create and submit storage mode form with a field hidden by logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics } = TEST_FIELD_HIDDEN_BY_LOGIC - const formSettings = getSettings() - - // Test - await runEncryptSubmissionTest(page, Form, { - formFields, - formLogics, - formSettings, - }) - }) - - test('Create storage mode form with submission disabled by chained logic', async ({ - page, - }) => { - // Define - const { formFields, formLogics, preventSubmitMessage } = - TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC - const formSettings = getSettings() - - // Test - const { form } = await createForm(page, Form, FormResponseMode.Encrypt, { - formFields, - formLogics, - formSettings, - }) - await verifySubmissionDisabled( - page, - { form, formFields, formSettings }, - preventSubmitMessage, - ) - await deleteDocById(Form, form._id) - }) -}) From 0399901bdf037593b9a962010398ef10af7c3baf Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Sat, 29 Apr 2023 10:07:59 +0800 Subject: [PATCH 28/55] ci: break playwright tests into different steps --- .github/workflows/playwright.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 9ef9e184db..f6a6ad13ee 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,8 +22,10 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build - - name: Run Playwright tests - run: npx playwright test + - name: Run Playwright tests (login) + run: npx playwright test __tests__/e2e/login.spec.ts + - name: Run Playwright tests (email-submission) + run: npx playwright test __tests__/e2e/email-submission.spec.ts - uses: actions/upload-artifact@v3 if: always() with: From 971a41d469a6c3c84f44933fadde01977a3cedbb Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 10:17:23 +0800 Subject: [PATCH 29/55] ci: break playwright tests into different steps --- .github/workflows/playwright.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f6a6ad13ee..ae67bfdd46 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -6,7 +6,7 @@ on: branches: [develop, release-al2, master] jobs: test: - timeout-minutes: 120 + timeout-minutes: 90 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -24,8 +24,28 @@ jobs: run: npm run build - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - - name: Run Playwright tests (email-submission) - run: npx playwright test __tests__/e2e/email-submission.spec.ts + - name: Run Playwright tests (email-submission-1) + run: -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (email-submission-2) + run: -g "Create and submit email mode form with all fields optional" + - name: Run Playwright tests (email-submission-3) + run: -g "Create and submit email mode form with identical attachment names" + - name: Run Playwright tests (email-submission-4) + run: -g "Create and submit email mode form with optional and required attachments" + - name: Run Playwright tests (email-submission-5) + run: -g "Create and submit email mode form with Singpass authentication" + - name: Run Playwright tests (email-submission-6) + run: -g "Create and submit email mode form with Corppass authentication" + - name: Run Playwright tests (email-submission-7) + run: -g "Create and submit email mode form with SGID authentication" + - name: Run Playwright tests (email-submission-8) + run: -g "Create and submit email mode form with MyInfo fields" + - name: Run Playwright tests (email-submission-9) + run: -g "Create and submit email mode form with all fields shown by logic" + - name: Run Playwright tests (email-submission-10) + run: -g "Create and submit email mode form with a field hidden by logic" + - name: Run Playwright tests (email-submission-11) + run: -g "Create email mode form with submission disabled by chained logic" - uses: actions/upload-artifact@v3 if: always() with: From 78f6fb1fafd3658c2f3e22f925879238011b0f1c Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 10:31:40 +0800 Subject: [PATCH 30/55] fix: forgot to add the commands themselves :/ --- .github/workflows/playwright.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index ae67bfdd46..6b0fc9a25c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -25,27 +25,27 @@ jobs: - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - name: Run Playwright tests (email-submission-1) - run: -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) - run: -g "Create and submit email mode form with all fields optional" + run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) - run: -g "Create and submit email mode form with identical attachment names" + run: npx playwright test -g "Create and submit email mode form with identical attachment names" - name: Run Playwright tests (email-submission-4) - run: -g "Create and submit email mode form with optional and required attachments" + run: npx playwright test -g "Create and submit email mode form with optional and required attachments" - name: Run Playwright tests (email-submission-5) - run: -g "Create and submit email mode form with Singpass authentication" + run: npx playwright test -g "Create and submit email mode form with Singpass authentication" - name: Run Playwright tests (email-submission-6) - run: -g "Create and submit email mode form with Corppass authentication" + run: npx playwright test -g "Create and submit email mode form with Corppass authentication" - name: Run Playwright tests (email-submission-7) - run: -g "Create and submit email mode form with SGID authentication" + run: npx playwright test -g "Create and submit email mode form with SGID authentication" - name: Run Playwright tests (email-submission-8) - run: -g "Create and submit email mode form with MyInfo fields" + run: npx playwright test -g "Create and submit email mode form with MyInfo fields" - name: Run Playwright tests (email-submission-9) - run: -g "Create and submit email mode form with all fields shown by logic" + run: npx playwright test -g "Create and submit email mode form with all fields shown by logic" - name: Run Playwright tests (email-submission-10) - run: -g "Create and submit email mode form with a field hidden by logic" + run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) - run: -g "Create email mode form with submission disabled by chained logic" + run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - uses: actions/upload-artifact@v3 if: always() with: From 44dd1caeeeed82f33d3bc07490ed25f646da1303 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 11:16:30 +0800 Subject: [PATCH 31/55] ci: move first email test to the end --- .github/workflows/playwright.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 6b0fc9a25c..191dfb866b 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,10 +22,6 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build - - name: Run Playwright tests (login) - run: npx playwright test __tests__/e2e/login.spec.ts - - name: Run Playwright tests (email-submission-1) - run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) @@ -46,6 +42,10 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" + - name: Run Playwright tests (email-submission-1) + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (login) + run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 if: always() with: From f79faa997f354910b9dd1be68c2638c2fb8ad1fd Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 20:52:14 +0800 Subject: [PATCH 32/55] ci: extract getByRole to separate step --- .github/workflows/playwright.yml | 4 ++-- __tests__/e2e/helpers/createForm.ts | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 191dfb866b..4ebc1ad092 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,6 +22,8 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build + - name: Run Playwright tests (email-submission-1) + run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (email-submission-2) run: npx playwright test -g "Create and submit email mode form with all fields optional" - name: Run Playwright tests (email-submission-3) @@ -42,8 +44,6 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - - name: Run Playwright tests (email-submission-1) - run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 50fa6cdbc2..aceacada8f 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -459,15 +459,13 @@ const addBasicField = async ( // Handle the rest of the individual fields. switch (field.fieldType) { - case BasicField.Attachment: - await fillDropdown( - page, - page.getByRole('textbox', { - name: 'Maximum size of individual attachment', - }), - `${field.attachmentSize} MB`, - ) + case BasicField.Attachment: { + const textBox = page.getByRole('textbox', { + name: 'Maximum size of individual attachment', + }) + await fillDropdown(page, textBox, `${field.attachmentSize} MB`) break + } case BasicField.Checkbox: if (field.validateByValue) { await page.getByLabel('Selection limits').click() From 0c05e281173c9de531a44c655670db36f93c882f Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 20:57:27 +0800 Subject: [PATCH 33/55] ci: temporarily use only 1 browser to debug faster --- playwright.config.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index 17c5f82b3f..fe0faeb02a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -50,19 +50,19 @@ const config: PlaywrightTestConfig = { ...devices['Desktop Chrome'], }, }, - { - name: 'firefox', - use: { - ...devices['Desktop Firefox'], - }, - }, + // { + // name: 'firefox', + // use: { + // ...devices['Desktop Firefox'], + // }, + // }, - { - name: 'webkit', - use: { - ...devices['Desktop Safari'], - }, - }, + // { + // name: 'webkit', + // use: { + // ...devices['Desktop Safari'], + // }, + // }, /* Test against mobile viewports. */ // { From 1a7d18fe1bbf9070aeb5b4024c57494162e943e3 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 22:57:38 +0800 Subject: [PATCH 34/55] ci: check if combobox was the deal --- __tests__/e2e/helpers/createForm.ts | 14 ++++++++------ __tests__/e2e/utils/field.ts | 8 ++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index aceacada8f..67fcf82c67 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -459,13 +459,15 @@ const addBasicField = async ( // Handle the rest of the individual fields. switch (field.fieldType) { - case BasicField.Attachment: { - const textBox = page.getByRole('textbox', { - name: 'Maximum size of individual attachment', - }) - await fillDropdown(page, textBox, `${field.attachmentSize} MB`) + case BasicField.Attachment: + await fillDropdown( + page, + page.getByRole('combobox', { + name: 'Maximum size of individual attachment', + }), + `${field.attachmentSize} MB`, + ) break - } case BasicField.Checkbox: if (field.validateByValue) { await page.getByLabel('Selection limits').click() diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index 12d5b88582..a9ba7676b8 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -167,12 +167,8 @@ export const fillDropdown = async ( input: Locator, value: string, ): Promise => { - await input.fill(value) - const menuId = await input.getAttribute('aria-controls') - const menu = page.locator(`id=${menuId}`) - // Scroll menu into view to avoid flakiness. - await menu.scrollIntoViewIfNeeded() - await menu.getByRole('option', { name: value }).click() + await input.click() + await page.getByRole('option', { name: value }).click() } /** From 62005cf38420ff4b4f5aada0f43aa299dc7b2663 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 2 May 2023 23:24:38 +0800 Subject: [PATCH 35/55] ci: commit to combobox for fillDropdown input --- __tests__/e2e/helpers/createForm.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 67fcf82c67..4d076cf426 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -593,12 +593,12 @@ const addBasicField = async ( case BasicField.Rating: await fillDropdown( page, - page.getByRole('textbox', { name: 'Number of steps' }), + page.getByRole('combobox', { name: 'Number of steps' }), String(field.ratingOptions.steps), ) await fillDropdown( page, - page.getByRole('textbox', { name: 'Shape' }), + page.getByRole('combobox', { name: 'Shape' }), field.ratingOptions.shape, ) break From 2bd6ea02f5293c25d3ebbcf304fbcffd7689089a Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 00:29:42 +0800 Subject: [PATCH 36/55] fix: change id=show locator to combobox --- __tests__/e2e/helpers/createForm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 4d076cf426..b867c6025d 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -721,7 +721,7 @@ const addLogics = async ( await fillDropdown(page, logicTypeInput, 'Show field(s)') await fillMultiDropdown( page, - page.locator('id=show'), + page.getByRole('combobox', { name: 'Show' }), logic.show.map((n) => getTitleWithQuestionNumber(formFields, n)), ) break From 84ea65b2da31aa9d47feae5ca443b205f48a7bfc Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 01:15:59 +0800 Subject: [PATCH 37/55] fix: use getByRole to pick logic type --- __tests__/e2e/helpers/createForm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index b867c6025d..886b7ea8fe 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -715,7 +715,7 @@ const addLogics = async ( } } - const logicTypeInput = page.locator('id=logicType') + const logicTypeInput = page.getByRole('combobox', { name: 'Then' }) switch (logic.logicType) { case LogicType.ShowFields: await fillDropdown(page, logicTypeInput, 'Show field(s)') From c352f0e6d779752a3dc770a9e6a1cbe081c9f8ce Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:09:10 +0800 Subject: [PATCH 38/55] fix: remove closing of options in fillMultiDropdown --- __tests__/e2e/utils/field.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index a9ba7676b8..45c3ca68b4 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -183,7 +183,9 @@ export const fillMultiDropdown = async ( input: Locator, values: string[], ): Promise => { - for (const value of values) await fillDropdown(page, input, value) + await input.click() + for (const value of values) + await page.getByRole('option', { name: value }).click() // Multiselect dropdown, click the input again to close the popover await input.click() } From 2480de4144f5392bd272ed311ce8ea211fd46d03 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:10:41 +0800 Subject: [PATCH 39/55] ci: add enumerated encrypt-submission tests --- .github/workflows/playwright.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 4ebc1ad092..c23ed81ddf 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -44,6 +44,16 @@ jobs: run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - name: Run Playwright tests (email-submission-11) run: npx playwright test -g "Create email mode form with submission disabled by chained logic" + - name: Run Playwright tests (encrypt-submission-1) + run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (encrypt-submission-2) + run: npx playwright test -g "Create and submit storage mode form with all fields optional" + - name: Run Playwright tests (encrypt-submission-3) + run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" + - name: Run Playwright tests (encrypt-submission-4) + run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" + - name: Run Playwright tests (encrypt-submission-5) + run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts - uses: actions/upload-artifact@v3 From 066fd2827ad270182b13d00bb7577def07ac13ad Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Wed, 3 May 2023 10:13:55 +0800 Subject: [PATCH 40/55] ci: put encrypt-submission playwright tests back --- __tests__/e2e/encrypt-submission.spec.ts | 137 +++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 __tests__/e2e/encrypt-submission.spec.ts diff --git a/__tests__/e2e/encrypt-submission.spec.ts b/__tests__/e2e/encrypt-submission.spec.ts new file mode 100644 index 0000000000..e1516a2b59 --- /dev/null +++ b/__tests__/e2e/encrypt-submission.spec.ts @@ -0,0 +1,137 @@ +import mongoose from 'mongoose' +import { BasicField, FormResponseMode } from 'shared/types' + +import { IFormModel } from 'src/types' + +import { + ALL_FIELDS as _ALL_FIELDS, + NO_LOGIC, + TEST_ALL_FIELDS_SHOWN_BY_LOGIC, + TEST_FIELD_HIDDEN_BY_LOGIC, + TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC, +} from './constants' +import { test } from './fixtures' +import { + createForm, + createSubmissionTestRunnerForResponseMode, + verifySubmissionDisabled, +} from './helpers' +import { + createBlankVersion, + createOptionalVersion, + deleteDocById, + getSettings, + makeModel, + makeMongooseFixtures, +} from './utils' + +// TODO: Attachment fields don't work on storage mode unless we spin up localstack. +const ALL_FIELDS = _ALL_FIELDS.filter( + (field) => field.fieldType !== BasicField.Attachment, +) + +const runEncryptSubmissionTest = createSubmissionTestRunnerForResponseMode( + FormResponseMode.Encrypt, +) + +let db: mongoose.Connection +let Form: IFormModel + +test.describe('Storage form submission', () => { + test.beforeAll(async () => { + // Create models + db = await makeMongooseFixtures() + Form = makeModel(db, 'form.server.model', 'Form') + }) + test.afterAll(async () => { + // Clean up db + db.models = {} + await db.close() + }) + + test('Create and submit storage mode form with all fields', async ({ + page, + }) => { + // Define + const formFields = ALL_FIELDS + const formLogics = NO_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with all fields optional', async ({ + page, + }) => { + // Define + const formFields = ALL_FIELDS.map((ff) => + createBlankVersion(createOptionalVersion(ff)), + ) + const formLogics = NO_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with all fields shown by logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics } = TEST_ALL_FIELDS_SHOWN_BY_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create and submit storage mode form with a field hidden by logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics } = TEST_FIELD_HIDDEN_BY_LOGIC + const formSettings = getSettings() + + // Test + await runEncryptSubmissionTest(page, Form, { + formFields, + formLogics, + formSettings, + }) + }) + + test('Create storage mode form with submission disabled by chained logic', async ({ + page, + }) => { + // Define + const { formFields, formLogics, preventSubmitMessage } = + TEST_SUBMISSION_DISABLED_BY_CHAINED_LOGIC + const formSettings = getSettings() + + // Test + const { form } = await createForm(page, Form, FormResponseMode.Encrypt, { + formFields, + formLogics, + formSettings, + }) + await verifySubmissionDisabled( + page, + { form, formFields, formSettings }, + preventSubmitMessage, + ) + await deleteDocById(Form, form._id) + }) +}) From 682d1d29831e13d627848d9affd925c1f4a3f58d Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 9 May 2023 17:54:49 +0800 Subject: [PATCH 41/55] fix: add ssn env site name for playwright --- __tests__/e2e/setup/.test-env | 1 + 1 file changed, 1 insertion(+) diff --git a/__tests__/e2e/setup/.test-env b/__tests__/e2e/setup/.test-env index 61d70c685b..036de66fb5 100644 --- a/__tests__/e2e/setup/.test-env +++ b/__tests__/e2e/setup/.test-env @@ -73,6 +73,7 @@ APP_URL=http://localhost:5000 SECRET_ENV=development INTRANET_IP_LIST_PATH=tests/mock-intranet-ips.txt +SSN_ENV_SITE_NAME=test REACT_SWITCH_ENV_FEEDBACK_FORM_ID_ADMIN=62da6a569ee8e90143b5da26 REACT_SWITCH_ENV_FEEDBACK_FORM_ID_RESPONDENT=62da6a569ee8e90143b5da26 From 3af498ee26d0b8c9f0246f2052cd3368e7042075 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Tue, 9 May 2023 18:49:32 +0800 Subject: [PATCH 42/55] fix: typo for SSM_ENV_SITE_NAME --- __tests__/e2e/setup/.test-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/e2e/setup/.test-env b/__tests__/e2e/setup/.test-env index 036de66fb5..cad1b201c1 100644 --- a/__tests__/e2e/setup/.test-env +++ b/__tests__/e2e/setup/.test-env @@ -73,7 +73,7 @@ APP_URL=http://localhost:5000 SECRET_ENV=development INTRANET_IP_LIST_PATH=tests/mock-intranet-ips.txt -SSN_ENV_SITE_NAME=test +SSM_ENV_SITE_NAME=test REACT_SWITCH_ENV_FEEDBACK_FORM_ID_ADMIN=62da6a569ee8e90143b5da26 REACT_SWITCH_ENV_FEEDBACK_FORM_ID_RESPONDENT=62da6a569ee8e90143b5da26 From 83150f5e779713540b943da4168ce880ddc28fbd Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 17:17:41 +0800 Subject: [PATCH 43/55] fix: inputLocator for fillMultiDropdown --- __tests__/e2e/helpers/createForm.ts | 4 +++- __tests__/e2e/utils/field.ts | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 72f821afa6..e45440cda7 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -719,7 +719,9 @@ const addLogics = async ( await fillDropdown(page, logicTypeInput, 'Show field(s)') await fillMultiDropdown( page, - page.getByLabel('Show').first(), + page + .getByRole('group') + .filter({ has: page.getByLabel('Show').first() }), logic.show.map((n) => getTitleWithQuestionNumber(formFields, n)), ) break diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index 6e15bd3845..fd1ada001b 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -180,12 +180,10 @@ export const fillDropdown = async ( */ export const fillMultiDropdown = async ( page: Page, - input: Locator, + inputScope: Locator, values: string[], ): Promise => { - const inputGroup = page.getByRole('group').filter({ has: input }) - - await inputGroup + await inputScope .getByRole('button', { name: 'Open dropdown options' }) .click() for (const value of values) { @@ -193,7 +191,7 @@ export const fillMultiDropdown = async ( await option.scrollIntoViewIfNeeded() await option.click() } - await inputGroup + await inputScope .getByRole('button', { name: 'Close dropdown options' }) .click() } From 23e658027a31116b36af09ce88c1c5a6b5d73208 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 17:57:42 +0800 Subject: [PATCH 44/55] fix: get parent group for conditions multiDropdown --- __tests__/e2e/helpers/createForm.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index e45440cda7..9d352ccb5c 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -695,7 +695,11 @@ const addLogics = async ( const valueInput = page.locator(`id=conditions.${i}.value`) switch (state) { case LogicConditionState.Either: - await fillMultiDropdown(page, valueInput, value) + await fillMultiDropdown( + page, + page.getByRole('group').filter({ has: valueInput }), + value, + ) break default: switch (formFields[field].fieldType) { From b04d782bddd8c5657cbfcaef3b2a08d3dd489b7a Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 21:52:36 +0800 Subject: [PATCH 45/55] fix: click first button for downshift found --- __tests__/e2e/utils/field.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index fd1ada001b..4f2ac19943 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -185,6 +185,7 @@ export const fillMultiDropdown = async ( ): Promise => { await inputScope .getByRole('button', { name: 'Open dropdown options' }) + .first() .click() for (const value of values) { const option = page.getByRole('option', { name: value }) @@ -193,6 +194,7 @@ export const fillMultiDropdown = async ( } await inputScope .getByRole('button', { name: 'Close dropdown options' }) + .first() .click() } From d4e7373c024a904d261ed016bcf41850dc01e951 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 22:15:04 +0800 Subject: [PATCH 46/55] fix: try last instead for dropdown --- __tests__/e2e/utils/field.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__tests__/e2e/utils/field.ts b/__tests__/e2e/utils/field.ts index 4f2ac19943..5aecda775f 100644 --- a/__tests__/e2e/utils/field.ts +++ b/__tests__/e2e/utils/field.ts @@ -185,7 +185,7 @@ export const fillMultiDropdown = async ( ): Promise => { await inputScope .getByRole('button', { name: 'Open dropdown options' }) - .first() + .last() .click() for (const value of values) { const option = page.getByRole('option', { name: value }) @@ -194,7 +194,7 @@ export const fillMultiDropdown = async ( } await inputScope .getByRole('button', { name: 'Close dropdown options' }) - .first() + .last() .click() } From 08dff29f73e562e392dca853f94c77413c976ac1 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 23:01:11 +0800 Subject: [PATCH 47/55] ci: restore commented browsers --- playwright.config.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index fe0faeb02a..17c5f82b3f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -50,19 +50,19 @@ const config: PlaywrightTestConfig = { ...devices['Desktop Chrome'], }, }, - // { - // name: 'firefox', - // use: { - // ...devices['Desktop Firefox'], - // }, - // }, + { + name: 'firefox', + use: { + ...devices['Desktop Firefox'], + }, + }, - // { - // name: 'webkit', - // use: { - // ...devices['Desktop Safari'], - // }, - // }, + { + name: 'webkit', + use: { + ...devices['Desktop Safari'], + }, + }, /* Test against mobile viewports. */ // { From 6a1943af740888d86055facf57f26aa9821b2bcd Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 23:02:34 +0800 Subject: [PATCH 48/55] ci: reduce number of playwright retries to 1 --- playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index 17c5f82b3f..cca21a3c69 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -26,7 +26,7 @@ const config: PlaywrightTestConfig = { /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, + retries: process.env.CI ? 1 : 0, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ From 19d08f4ee383c1e499ca5cf8d50467c2dedff3ef Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Thu, 11 May 2023 23:07:39 +0800 Subject: [PATCH 49/55] ci: playwright test suite for each step --- .github/workflows/playwright.yml | 36 ++++---------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 01797a4781..ae246625c7 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -22,40 +22,12 @@ jobs: NODE_OPTIONS: '--max-old-space-size=4096 --openssl-legacy-provider' REACT_APP_FORMSG_SDK_MODE: 'test' run: npm run build - - name: Run Playwright tests (encrypt-submission-1) - run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" - - name: Run Playwright tests (encrypt-submission-2) - run: npx playwright test -g "Create and submit storage mode form with all fields optional" - - name: Run Playwright tests (encrypt-submission-3) - run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" - - name: Run Playwright tests (encrypt-submission-4) - run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" - - name: Run Playwright tests (encrypt-submission-5) - run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - - name: Run Playwright tests (email-submission-1) - run: npx playwright test -g "Create and submit email mode form with all fields" -gv "optional|shown by logic" - - name: Run Playwright tests (email-submission-2) - run: npx playwright test -g "Create and submit email mode form with all fields optional" - - name: Run Playwright tests (email-submission-3) - run: npx playwright test -g "Create and submit email mode form with identical attachment names" - - name: Run Playwright tests (email-submission-4) - run: npx playwright test -g "Create and submit email mode form with optional and required attachments" - - name: Run Playwright tests (email-submission-5) - run: npx playwright test -g "Create and submit email mode form with Singpass authentication" - - name: Run Playwright tests (email-submission-6) - run: npx playwright test -g "Create and submit email mode form with Corppass authentication" - - name: Run Playwright tests (email-submission-7) - run: npx playwright test -g "Create and submit email mode form with SGID authentication" - - name: Run Playwright tests (email-submission-8) - run: npx playwright test -g "Create and submit email mode form with MyInfo fields" - - name: Run Playwright tests (email-submission-9) - run: npx playwright test -g "Create and submit email mode form with all fields shown by logic" - - name: Run Playwright tests (email-submission-10) - run: npx playwright test -g "Create and submit email mode form with a field hidden by logic" - - name: Run Playwright tests (email-submission-11) - run: npx playwright test -g "Create email mode form with submission disabled by chained logic" - name: Run Playwright tests (login) run: npx playwright test __tests__/e2e/login.spec.ts + - name: Run Playwright tests (email-submission) + run: npx playwright test __tests__/e2e/email-submission.spec.ts + - name: Run Playwright tests (encrypt-submission) + run: npx playwright test __tests__/e2e/encrypt-submission.spec.ts - uses: actions/upload-artifact@v3 if: always() with: From 9f3abe0a7f8e04e5b6c161f66a3e8a3231e52fc8 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 08:39:20 +0800 Subject: [PATCH 50/55] fix: adjust test timeouts - reduce wait timeout for popup modals to appear --- __tests__/e2e/helpers/createForm.ts | 2 +- playwright.config.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 9d352ccb5c..8d27ad7917 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -87,7 +87,7 @@ const addForm = async ( await page.goto(DASHBOARD_PAGE) // Wait for all modals to render - await page.waitForTimeout(1000) + await page.waitForTimeout(500) // Press escape 5 times to get rid of any banners await page.keyboard.press('Escape') diff --git a/playwright.config.ts b/playwright.config.ts index cca21a3c69..a1c5ae2820 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -13,7 +13,7 @@ dotenv.config({ const config: PlaywrightTestConfig = { testDir: './__tests__/e2e', /* Maximum time one test can run for. */ - timeout: 120 * 1000, + timeout: 180 * 1000, expect: { /** * Maximum time expect() should wait for the condition to be met. @@ -103,7 +103,7 @@ const config: PlaywrightTestConfig = { { command: 'npm run test:e2e-v2:server', url: 'http://localhost:5000/analytics/statistics', - timeout: 120 * 1000, + timeout: 180 * 1000, reuseExistingServer: !process.env.CI, }, ], From 96fffba319b3662cdb175fdb38f342f0176c8e14 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 09:41:34 +0800 Subject: [PATCH 51/55] ci: split storage mode playwright tests --- .github/workflows/playwright.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index ae246625c7..645d9c7fd7 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -26,6 +26,16 @@ jobs: run: npx playwright test __tests__/e2e/login.spec.ts - name: Run Playwright tests (email-submission) run: npx playwright test __tests__/e2e/email-submission.spec.ts + - name: Run Playwright tests (encrypt-submission-1) + run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" + - name: Run Playwright tests (encrypt-submission-2) + run: npx playwright test -g "Create and submit storage mode form with all fields optional" + - name: Run Playwright tests (encrypt-submission-3) + run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" + - name: Run Playwright tests (encrypt-submission-4) + run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" + - name: Run Playwright tests (encrypt-submission-5) + run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (encrypt-submission) run: npx playwright test __tests__/e2e/encrypt-submission.spec.ts - uses: actions/upload-artifact@v3 From 27353a371c6e60840bf97b9a73a67135e846c73e Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 14:49:56 +0800 Subject: [PATCH 52/55] ci: rm webkit from playwright tests temporarily --- playwright.config.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index a1c5ae2820..96eae2b314 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -57,12 +57,12 @@ const config: PlaywrightTestConfig = { }, }, - { - name: 'webkit', - use: { - ...devices['Desktop Safari'], - }, - }, + // { + // name: 'webkit', + // use: { + // ...devices['Desktop Safari'], + // }, + // }, /* Test against mobile viewports. */ // { From 7d125b7bc3c03868b4feab3ad6a19a6b3a17454f Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 16:43:32 +0800 Subject: [PATCH 53/55] ci: restore webServer timeout for playwright to 120 * 1000 --- playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index 96eae2b314..9996b9914f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -103,7 +103,7 @@ const config: PlaywrightTestConfig = { { command: 'npm run test:e2e-v2:server', url: 'http://localhost:5000/analytics/statistics', - timeout: 180 * 1000, + timeout: 120 * 1000, reuseExistingServer: !process.env.CI, }, ], From 01aebdadc6691076f2e88a7227eaa6e50ca71eda Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 16:46:20 +0800 Subject: [PATCH 54/55] ci: make encrypt-submissions playwright tests 1 step --- .github/workflows/playwright.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 645d9c7fd7..ae246625c7 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -26,16 +26,6 @@ jobs: run: npx playwright test __tests__/e2e/login.spec.ts - name: Run Playwright tests (email-submission) run: npx playwright test __tests__/e2e/email-submission.spec.ts - - name: Run Playwright tests (encrypt-submission-1) - run: npx playwright test -g "Create and submit storage mode form with all fields" -gv "optional|shown by logic" - - name: Run Playwright tests (encrypt-submission-2) - run: npx playwright test -g "Create and submit storage mode form with all fields optional" - - name: Run Playwright tests (encrypt-submission-3) - run: npx playwright test -g "Create and submit storage mode form with all fields shown by logic" - - name: Run Playwright tests (encrypt-submission-4) - run: npx playwright test -g "Create and submit storage mode form with a field hidden by logic" - - name: Run Playwright tests (encrypt-submission-5) - run: npx playwright test -g "Create storage mode form with submission disabled by chained logic" - name: Run Playwright tests (encrypt-submission) run: npx playwright test __tests__/e2e/encrypt-submission.spec.ts - uses: actions/upload-artifact@v3 From 87c5a4013a00e8e7f71953ca14e86ef96025e9a8 Mon Sep 17 00:00:00 2001 From: LinHuiqing Date: Fri, 12 May 2023 19:52:16 +0800 Subject: [PATCH 55/55] fix: use combobox for field settings with dropdown --- __tests__/e2e/helpers/createForm.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/__tests__/e2e/helpers/createForm.ts b/__tests__/e2e/helpers/createForm.ts index 8d27ad7917..fb6fe3ea66 100644 --- a/__tests__/e2e/helpers/createForm.ts +++ b/__tests__/e2e/helpers/createForm.ts @@ -462,7 +462,9 @@ const addBasicField = async ( case BasicField.Attachment: await fillDropdown( page, - page.getByLabel('Maximum size of individual attachment').first(), + page.getByRole('combobox', { + name: 'Maximum size of individual attachment', + }), `${field.attachmentSize} MB`, ) break @@ -591,12 +593,16 @@ const addBasicField = async ( case BasicField.Rating: await fillDropdown( page, - page.getByLabel('Number of steps').first(), + page.getByRole('combobox', { + name: 'Number of steps', + }), String(field.ratingOptions.steps), ) await fillDropdown( page, - page.getByLabel('Shape').first(), + page.getByRole('combobox', { + name: 'Shape', + }), field.ratingOptions.shape, ) break