test: add initial webview2 tests (#16827)

.github/workflows/tests_webview2.yml

@@ -0,0 +1,48 @@
+name: "WebView2 Tests"
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request:
+    paths-ignore:
+      - 'browser_patches/**'
+      - 'docs/**'
+    branches:
+      - main
+      - release-*
+
+env:
+  # Force terminal colors. @see https://www.npmjs.com/package/colors
+  FORCE_COLOR: 1
+  FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}
+
+jobs:
+  test_webview2:
+    name: WebView2
+    runs-on: windows-2022
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+    - uses: actions/setup-dotnet@v2
+      with:
+        dotnet-version: '6.0.x'
+    - run: npm i -g npm@8
+    - run: npm ci
+      env:
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    - run: npm run build
+    - run: dotnet build
+      working-directory: tests/webview2/webview2-app/
+    - run: npm run webview2test
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always()
+      shell: bash
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: webview2-test-results
+        path: test-results

package.json

@@ -18,6 +18,7 @@
     "wtest": "playwright test --config=tests/library/playwright.config.ts --project=webkit",
     "atest": "playwright test --config=tests/android/playwright.config.ts",
     "etest": "playwright test --config=tests/electron/playwright.config.ts",
+    "webview2test": "playwright test --config=tests/webview2/playwright.config.ts",
     "itest": "playwright test --config=tests/installation/playwright.config.ts",
     "stest": "playwright test --config=tests/stress/playwright.config.ts",
     "test-html-reporter": "playwright test --config=packages/html-reporter",

tests/android/androidTest.ts

@@ -50,6 +50,7 @@ export const androidTest = baseTest.extend<PageTestFixtures, AndroidWorkerFixtur
 
   isAndroid: [true, { scope: 'worker' }],
   isElectron: [false, { scope: 'worker' }],
+  isWebView2: [false, { scope: 'worker' }],
 
   androidContext: [async ({ androidDevice }, run) => {
     const context = await androidDevice.launchBrowser();

tests/config/browserTest.ts

@@ -81,6 +81,7 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
 
   isAndroid: [false, { scope: 'worker' }],
   isElectron: [false, { scope: 'worker' }],
+  isWebView2: [false, { scope: 'worker' }],
 
   contextFactory: async ({ _contextFactory }: any, run) => {
     await run(_contextFactory);

tests/page/locator-misc-2.spec.ts

@@ -43,8 +43,8 @@ it('should scroll into view', async ({ page, server, isAndroid }) => {
   }
 });
 
-it('should scroll zero-sized element into view', async ({ page, isAndroid, isElectron, browserName, isMac }) => {
-  it.fixme(isAndroid || isElectron);
+it('should scroll zero-sized element into view', async ({ page, isAndroid, isElectron, isWebView2, browserName, isMac }) => {
+  it.fixme(isAndroid || isElectron || isWebView2);
   it.skip(browserName === 'webkit' && isMac && parseInt(os.release(), 10) < 20, 'WebKit for macOS 10.15 is frozen.');
 
   await page.setContent(`

tests/page/page-basic.spec.ts

@@ -17,7 +17,9 @@
 
 import { test as it, expect } from './pageTest';
 
-it('should reject all promises when page is closed', async ({ page }) => {
+it('should reject all promises when page is closed', async ({ page, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   let error = null;
   await Promise.all([
     page.evaluate(() => new Promise(r => {})).catch(e => error = e),
@@ -26,14 +28,17 @@ it('should reject all promises when page is closed', async ({ page }) => {
   expect(error.message).toContain('Target closed');
 });
 
-it('should set the page close state', async ({ page }) => {
+it('should set the page close state', async ({ page, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   expect(page.isClosed()).toBe(false);
   await page.close();
   expect(page.isClosed()).toBe(true);
 });
 
-it('should pass page to close event', async ({ page, isAndroid }) => {
+it('should pass page to close event', async ({ page, isAndroid, isWebView2 }) => {
   it.fixme(isAndroid);
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
 
   const [closedPage] = await Promise.all([
     page.waitForEvent('close'),
@@ -42,8 +47,10 @@ it('should pass page to close event', async ({ page, isAndroid }) => {
   expect(closedPage).toBe(page);
 });
 
-it('should terminate network waiters', async ({ page, server, isAndroid }) => {
+it('should terminate network waiters', async ({ page, server, isAndroid, isWebView2 }) => {
   it.fixme(isAndroid);
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   const results = await Promise.all([
     page.waitForRequest(server.EMPTY_PAGE).catch(e => e),
     page.waitForResponse(server.EMPTY_PAGE).catch(e => e),
@@ -56,7 +63,9 @@ it('should terminate network waiters', async ({ page, server, isAndroid }) => {
   }
 });
 
-it('should be callable twice', async ({ page }) => {
+it('should be callable twice', async ({ page, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   await Promise.all([
     page.close(),
     page.close(),
@@ -90,7 +99,9 @@ it('should provide access to the opener page', async ({ page }) => {
   expect(opener).toBe(page);
 });
 
-it('should return null if parent page has been closed', async ({ page }) => {
+it('should return null if parent page has been closed', async ({ page, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   const [popup] = await Promise.all([
     page.waitForEvent('popup'),
     page.evaluate(() => window.open('about:blank')),
@@ -122,7 +133,8 @@ it('should pass self as argument to load event', async ({ page }) => {
   expect(eventArg).toBe(page);
 });
 
-it('should fail with error upon disconnect', async ({ page, isAndroid }) => {
+it('should fail with error upon disconnect', async ({ page, isAndroid, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
   it.fixme(isAndroid);
 
   let error;
@@ -161,7 +173,9 @@ it('page.close should work with window.close', async function({ page }) {
   await closedPromise;
 });
 
-it('page.close should work with page.close', async function({ page }) {
+it('page.close should work with page.close', async function({ page, isWebView2 }) {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   const closedPromise = new Promise(x => page.on('close', x));
   await page.close();
   await closedPromise;

tests/page/page-click.spec.ts

@@ -70,7 +70,9 @@ it('should click on a span with an inline element inside', async ({ page }) => {
   expect(await page.evaluate('CLICKED')).toBe(42);
 });
 
-it('should not throw UnhandledPromiseRejection when page closes', async ({ page }) => {
+it('should not throw UnhandledPromiseRejection when page closes', async ({ page, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   await Promise.all([
     page.close(),
     page.mouse.click(1, 2),

tests/page/page-close.spec.ts

@@ -17,6 +17,8 @@
 
 import { test as it, expect } from './pageTest';
 
+it.skip(({ isWebView2 }) => isWebView2, 'Page.close() is not supported in WebView2');
+
 it('should close page with active dialog', async ({ page }) => {
   await page.setContent(`<button onclick="setTimeout(() => alert(1))">alert</button>`);
   page.click('button');

tests/page/page-evaluate.spec.ts

@@ -360,7 +360,7 @@ it('shuld properly serialize window.performance object', async ({ page }) => {
   expect(await page.evaluate(() => performance)).toEqual({
     'navigation': {
       'redirectCount': 0,
-      'type': 0
+      'type': expect.any(Number),
     },
     'timeOrigin': expect.any(Number),
     'timing': {

tests/page/page-event-crash.spec.ts

@@ -68,9 +68,10 @@ it.describe('', () => {
     expect(error.message).toContain('Navigation failed because page crashed');
   });
 
-  it('should be able to close context when page crashes', async ({ isAndroid, isElectron, page, toImpl, browserName, platform, mode }) => {
+  it('should be able to close context when page crashes', async ({ isAndroid, isElectron, isWebView2, page, toImpl, browserName, platform, mode }) => {
     it.skip(isAndroid);
     it.skip(isElectron);
+    it.skip(isWebView2, 'Page.close() is not supported in WebView2');
 
     await page.setContent(`<div>This page should crash</div>`);
     crash({ page, toImpl, browserName, platform, mode });

tests/page/page-event-popup.spec.ts

@@ -144,7 +144,9 @@ it('should work with clicking target=_blank and rel=noopener', async ({ page, se
   expect(await popup.evaluate(() => !!window.opener)).toBe(false);
 });
 
-it('should not treat navigations as new popups', async ({ page, server }) => {
+it('should not treat navigations as new popups', async ({ page, server, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   await page.goto(server.EMPTY_PAGE);
   await page.setContent('<a target=_blank rel=noopener href="/one-style.html">yo</a>');
   const [popup] = await Promise.all([

tests/page/page-expose-function.spec.ts

@@ -221,8 +221,9 @@ it('exposeBindingHandle should throw for multiple arguments', async ({ page }) =
   expect(error.message).toContain('exposeBindingHandle supports a single argument, 2 received');
 });
 
-it('should not result in unhandled rejection', async ({ page, isAndroid }) => {
+it('should not result in unhandled rejection', async ({ page, isAndroid, isWebView2 }) => {
   it.fixme(isAndroid);
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
 
   const closedPromise = page.waitForEvent('close');
   await page.exposeFunction('foo', async () => {

tests/page/page-request-continue.spec.ts

@@ -115,7 +115,9 @@ it('should not allow changing protocol when overriding url', async ({ page, serv
   expect(error.message).toContain('New URL must have same protocol as overridden URL');
 });
 
-it('should not throw when continuing while page is closing', async ({ page, server }) => {
+it('should not throw when continuing while page is closing', async ({ page, server, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   let done;
   await page.route('**/*', async route => {
     done = Promise.all([
@@ -128,7 +130,9 @@ it('should not throw when continuing while page is closing', async ({ page, serv
   expect(error).toBeInstanceOf(Error);
 });
 
-it('should not throw when continuing after page is closed', async ({ page, server }) => {
+it('should not throw when continuing after page is closed', async ({ page, server, isWebView2 }) => {
+  it.skip(isWebView2, 'Page.close() is not supported in WebView2');
+
   let done;
   await page.route('**/*', async route => {
     await page.close();

tests/page/page-screenshot.spec.ts

@@ -274,9 +274,10 @@ it.describe('page screenshot', () => {
     expect(screenshot).toMatchSnapshot('screenshot-canvas.png', { threshold: 0.4 });
   });
 
-  it('should capture canvas changes', async ({ page, isElectron, browserName, isMac }) => {
+  it('should capture canvas changes', async ({ page, isElectron, browserName, isMac, isWebView2 }) => {
     it.fixme(browserName === 'webkit' && isMac, 'https://github.com/microsoft/playwright/issues/8796,https://github.com/microsoft/playwright/issues/16180');
     it.skip(isElectron);
+    it.skip(isWebView2);
     await page.goto('data:text/html,<canvas></canvas>');
     await page.evaluate(() => {
       const canvas = document.querySelector('canvas');

tests/page/pageTest.ts

@@ -20,6 +20,7 @@ import type { TestModeTestFixtures, TestModeWorkerFixtures, TestModeWorkerOption
 import { androidTest } from '../android/androidTest';
 import { browserTest } from '../config/browserTest';
 import { electronTest } from '../electron/electronTest';
+import { webView2Test } from '../webview2/webView2Test';
 import type { PageTestFixtures, PageWorkerFixtures } from './pageTestApi';
 import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtures';
 export { expect } from '@playwright/test';
@@ -30,5 +31,7 @@ if (process.env.PWPAGE_IMPL === 'android')
   impl = androidTest;
 if (process.env.PWPAGE_IMPL === 'electron')
   impl = electronTest;
+if (process.env.PWPAGE_IMPL === 'webview2')
+  impl = webView2Test;
 
 export const test = impl;

tests/page/pageTestApi.ts

@@ -33,4 +33,5 @@ export type PageWorkerFixtures = {
   browserMajorVersion: number;
   isAndroid: boolean;
   isElectron: boolean;
+  isWebView2: boolean;
 };

tests/webview2/globalSetup.ts

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import childProcess from 'child_process';
+import playwright from 'playwright';
+
+export default async () => {
+  const cdpPort = 9876;
+  const spawnedProcess = childProcess.spawn(path.join(__dirname, 'webview2-app/bin/Debug/net6.0-windows/webview2.exe'), {
+    shell: true,
+    env: {
+      ...process.env,
+      WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS: `--remote-debugging-port=${cdpPort}`,
+    }
+  });
+  await new Promise<void>(resolve => spawnedProcess.stdout.on('data', (data: Buffer): void => {
+    if (data.toString().includes('WebView2 initialized'))
+      resolve();
+  }));
+  const browser = await playwright.chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`);
+  const chromeVersion = await browser.contexts()[0].pages()[0].evaluate(() => navigator.userAgent.match(/Chrome\/(.*?) /)[1]);
+  process.env.PWTEST_WEBVIEW2_CHROMIUM_VERSION = chromeVersion;
+  await browser.close();
+  childProcess.spawnSync(`taskkill /pid ${spawnedProcess.pid} /T /F`, { shell: true });
+};

tests/webview2/playwright.config.ts

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { config as loadEnv } from 'dotenv';
+loadEnv({ path: path.join(__dirname, '..', '..', '.env') });
+
+import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test';
+import * as path from 'path';
+import type { CoverageWorkerOptions } from '../config/coverageFixtures';
+
+process.env.PWPAGE_IMPL = 'webview2';
+
+const outputDir = path.join(__dirname, '..', '..', 'test-results');
+const testDir = path.join(__dirname, '..');
+const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & PlaywrightTestOptions> = {
+  testDir,
+  outputDir,
+  timeout: 30000,
+  globalTimeout: 5400000,
+  workers: process.env.CI ? 1 : undefined,
+  forbidOnly: !!process.env.CI,
+  preserveOutput: process.env.CI ? 'failures-only' : 'always',
+  retries: process.env.CI ? 3 : 0,
+  reporter: process.env.CI ? [
+    ['dot'],
+    ['json', { outputFile: path.join(outputDir, 'report.json') }],
+  ] : 'line',
+  projects: [],
+  globalSetup: './globalSetup.ts',
+};
+
+const metadata = {
+  platform: process.platform,
+  headful: true,
+  browserName: 'webview2',
+  channel: undefined,
+  mode: 'default',
+  video: false,
+};
+
+config.projects.push({
+  name: 'chromium',  // We use 'chromium' here to share screenshots with chromium.
+  use: {
+    browserName: 'chromium',
+    coverageName: 'webview2',
+  },
+  testDir: path.join(testDir, 'page'),
+  metadata,
+});
+
+export default config;