From e078bf192a9cc4caa294f7d7ac1391e32c4005fc Mon Sep 17 00:00:00 2001 From: Stepan Kuzmin Date: Mon, 15 May 2023 16:12:23 +0300 Subject: [PATCH] Use WebGL2 in render tests by default (#12701) * Use WebGL2 in render tests by default * Increase the `allowed` value for the heatmap render tests * Add WebGL1 Firefox render test * Log WebGL usage and expected image path --- .circleci/config.yml | 53 ++++++++++++++++--- test/integration/lib/render.js | 18 ++++--- .../render-tests/fog/2d/heatmap/style.json | 3 +- .../heatmap-color/default/style.json | 1 + .../heatmap-color/expression/style.json | 1 + .../heatmap-intensity/default/style.json | 1 + .../heatmap-intensity/function/style.json | 1 + .../heatmap-intensity/literal/style.json | 1 + .../heatmap-opacity/default/style.json | 1 + .../heatmap-radius/default/style.json | 1 + .../heatmap-radius/function/style.json | 1 + .../heatmap-radius/literal/style.json | 1 + .../heatmap-radius/pitch30/style.json | 1 + .../heatmap-weight/default/style.json | 1 + .../identity-property-function/style.json | 1 + .../heatmap-weight/literal/style.json | 1 + test/util/html_generator.js | 8 ++- 17 files changed, 78 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb5ab884002..c35d18e38ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ workflows: filters: tags: only: /.*/ - - test-render-linux-firefox-dev: + - test-render-linux-chrome-webgl1-dev: requires: - prepare-linux filters: @@ -139,6 +139,18 @@ workflows: filters: tags: only: /.*/ + - test-render-linux-firefox-dev: + requires: + - prepare-linux + filters: + tags: + only: /.*/ + - test-render-linux-firefox-webgl1-dev: + requires: + - prepare-linux + filters: + tags: + only: /.*/ - prepare-mac: filters: tags: @@ -149,7 +161,7 @@ workflows: filters: tags: only: /.*/ - - test-render-mac-chrome-webgl2-dev: + - test-render-mac-chrome-webgl1-dev: requires: - prepare-mac filters: @@ -161,7 +173,7 @@ workflows: filters: tags: only: /.*/ - - test-render-mac-safari-webgl2-dev: + - test-render-mac-safari-webgl1-dev: requires: - prepare-mac filters: @@ -423,6 +435,21 @@ jobs: - store_artifacts: path: "test/integration/render-tests/index.html" + test-render-linux-chrome-webgl1-dev: + <<: *linux-defaults + steps: + - attach_workspace: + at: ~/ + - browser-tools/install-chrome + - run: + name: Running tests in parallel + command: | + USE_WEBGL2=false yarn run test-render + - store_test_results: + path: test/integration/render-tests + - store_artifacts: + path: "test/integration/render-tests/index.html" + test-render-linux-chrome-prod: <<: *linux-defaults steps: @@ -459,6 +486,18 @@ jobs: - store_artifacts: path: "test/integration/render-tests/index.html" + test-render-linux-firefox-webgl1-dev: + <<: *linux-defaults + steps: + - attach_workspace: + at: ~/ + - browser-tools/install-firefox + - run: USE_WEBGL2=false yarn run test-render-firefox + - store_test_results: + path: test/integration/render-tests + - store_artifacts: + path: "test/integration/render-tests/index.html" + prepare-mac: <<: *mac-defaults steps: @@ -497,7 +536,7 @@ jobs: - store_artifacts: path: "test/integration/render-tests/index.html" - test-render-mac-chrome-webgl2-dev: + test-render-mac-chrome-webgl1-dev: <<: *mac-defaults parallelism: 3 steps: @@ -508,7 +547,7 @@ jobs: name: Creating test list command: | circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split --split-by=timings > tests-to-run.txt - - run: USE_WEBGL2=true yarn run test-render + - run: USE_WEBGL2=false yarn run test-render - store_test_results: path: test/integration/render-tests - store_artifacts: @@ -526,13 +565,13 @@ jobs: - store_artifacts: path: "test/integration/render-tests/index.html" - test-render-mac-safari-webgl2-dev: + test-render-mac-safari-webgl1-dev: <<: *mac-defaults resource_class: macos.m1.large.gen1 steps: - attach_workspace: at: ~/ - - run: USE_WEBGL2=true yarn run test-render-safari + - run: USE_WEBGL2=false yarn run test-render-safari - store_test_results: path: test/integration/render-tests - store_artifacts: diff --git a/test/integration/lib/render.js b/test/integration/lib/render.js index 8ffa802fb77..68b55ae94c5 100644 --- a/test/integration/lib/render.js +++ b/test/integration/lib/render.js @@ -20,6 +20,8 @@ import {vec3, vec4} from 'gl-matrix'; const browserWriteFile = new Worker('../util/browser_write_file.js'); +const useWebGL2 = process.env.USE_WEBGL2 ? process.env.USE_WEBGL2 !== 'false' : true; + // We are self-hosting test files. config.REQUIRE_ACCESS_TOKEN = false; window._suiteName = 'render-tests'; @@ -43,7 +45,7 @@ fakeCanvasContainer.style.top = '10px'; fakeCanvasContainer.style.left = '10px'; document.body.appendChild(fakeCanvasContainer); -setupHTML(); +setupHTML({useWebGL2}); const {canvas: expectedCanvas, ctx: expectedCtx} = createCanvas(); const {canvas: diffCanvas, ctx: diffCtx} = createCanvas(); @@ -129,13 +131,12 @@ function parseStyle(currentFixture) { return style; } -function parseOptions(currentFixture, style, useWebGL2) { +function parseOptions(currentFixture, style) { const options = { width: 512, height: 512, pixelRatio: 1, allowed: 0.00015, - useWebGL2, ...((style.metadata && style.metadata.test) || {}) }; @@ -192,7 +193,7 @@ async function renderMap(style, options) { attributionControl: false, preserveDrawingBuffer: true, axonometric: options.axonometric || false, - useWebGL2: options.useWebGL2 || false, + useWebGL2, skew: options.skew || [0, 0], fadeDuration: options.fadeDuration || 0, optimizeForTerrain: options.optimizeForTerrain || false, @@ -282,6 +283,7 @@ function getActualImageDataURL(actualImageData, map, {w, h}, options) { function calculateDiff(actualImageData, expectedImages, {w, h}) { // 2. draw expected.png into a canvas and extract ImageData + let minImageSrc; let minDiffImage; let minExpectedCanvas; let minDiff = Infinity; @@ -298,10 +300,11 @@ function calculateDiff(actualImageData, expectedImages, {w, h}) { minDiff = currentDiff; minDiffImage = diffImage; minExpectedCanvas = expectedCanvas; + minImageSrc = expectedImages[i].src; } } - return {minDiff, minDiffImage, minExpectedCanvas}; + return {minDiff, minDiffImage, minExpectedCanvas, minImageSrc}; } async function getActualImage(style, options) { @@ -323,7 +326,7 @@ async function runTest(t) { const expectedImages = await getExpectedImages(currentTestName, currentFixture); const style = parseStyle(currentFixture); - const options = parseOptions(currentFixture, style, process.env.USE_WEBGL2 && process.env.USE_WEBGL2 === 'true'); + const options = parseOptions(currentFixture, style); const {actualImageData, w, h} = await getActualImage(style, options); if (process.env.UPDATE) { @@ -335,7 +338,7 @@ async function runTest(t) { return; } - const {minDiff, minDiffImage, minExpectedCanvas} = calculateDiff(actualImageData, expectedImages, {w, h}); + const {minDiff, minDiffImage, minExpectedCanvas, minImageSrc} = calculateDiff(actualImageData, expectedImages, {w, h}); const pass = minDiff <= options.allowed; const testMetaData = { name: currentTestName, @@ -375,6 +378,7 @@ async function runTest(t) { // 7. pass image paths to testMetaData so the UI can render them testMetaData.actual = actual; testMetaData.expected = minExpectedCanvas.toDataURL(); + testMetaData.expectedPath = minImageSrc; testMetaData.imgDiff = imgDiff; } diff --git a/test/integration/render-tests/fog/2d/heatmap/style.json b/test/integration/render-tests/fog/2d/heatmap/style.json index 92c7a0a7463..4a84860c2eb 100644 --- a/test/integration/render-tests/fog/2d/heatmap/style.json +++ b/test/integration/render-tests/fog/2d/heatmap/style.json @@ -3,7 +3,8 @@ "metadata": { "test": { "height": 256, - "width": 256 + "width": 256, + "allowed": 0.00265 } }, "center": [ diff --git a/test/integration/render-tests/heatmap-color/default/style.json b/test/integration/render-tests/heatmap-color/default/style.json index f41200873b9..6845d1816a1 100644 --- a/test/integration/render-tests/heatmap-color/default/style.json +++ b/test/integration/render-tests/heatmap-color/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 64, "width": 256, + "allowed": 0.0073, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-color/expression/style.json b/test/integration/render-tests/heatmap-color/expression/style.json index 9beaa0dff6e..37c8d756a29 100644 --- a/test/integration/render-tests/heatmap-color/expression/style.json +++ b/test/integration/render-tests/heatmap-color/expression/style.json @@ -4,6 +4,7 @@ "test": { "height": 64, "width": 256, + "allowed": 0.0073, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-intensity/default/style.json b/test/integration/render-tests/heatmap-intensity/default/style.json index c7a936dad49..9c9303ccad9 100644 --- a/test/integration/render-tests/heatmap-intensity/default/style.json +++ b/test/integration/render-tests/heatmap-intensity/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.00593, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-intensity/function/style.json b/test/integration/render-tests/heatmap-intensity/function/style.json index 2492d147287..4f0a78020aa 100644 --- a/test/integration/render-tests/heatmap-intensity/function/style.json +++ b/test/integration/render-tests/heatmap-intensity/function/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.00263, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-intensity/literal/style.json b/test/integration/render-tests/heatmap-intensity/literal/style.json index f1be28bf29b..12d40e10a61 100644 --- a/test/integration/render-tests/heatmap-intensity/literal/style.json +++ b/test/integration/render-tests/heatmap-intensity/literal/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.0027, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-opacity/default/style.json b/test/integration/render-tests/heatmap-opacity/default/style.json index 4b4c3fb0711..a375d1d8960 100644 --- a/test/integration/render-tests/heatmap-opacity/default/style.json +++ b/test/integration/render-tests/heatmap-opacity/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 32, "width": 128, + "allowed": 0.0044, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-radius/default/style.json b/test/integration/render-tests/heatmap-radius/default/style.json index c7a936dad49..9c9303ccad9 100644 --- a/test/integration/render-tests/heatmap-radius/default/style.json +++ b/test/integration/render-tests/heatmap-radius/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.00593, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-radius/function/style.json b/test/integration/render-tests/heatmap-radius/function/style.json index 5ad88e9c009..1525da02f87 100644 --- a/test/integration/render-tests/heatmap-radius/function/style.json +++ b/test/integration/render-tests/heatmap-radius/function/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.01429, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-radius/literal/style.json b/test/integration/render-tests/heatmap-radius/literal/style.json index 7e092f0d72d..466ebf0edbc 100644 --- a/test/integration/render-tests/heatmap-radius/literal/style.json +++ b/test/integration/render-tests/heatmap-radius/literal/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.01164, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-radius/pitch30/style.json b/test/integration/render-tests/heatmap-radius/pitch30/style.json index ae024a3ac1e..b1d3e453f28 100644 --- a/test/integration/render-tests/heatmap-radius/pitch30/style.json +++ b/test/integration/render-tests/heatmap-radius/pitch30/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.02394, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-weight/default/style.json b/test/integration/render-tests/heatmap-weight/default/style.json index c7a936dad49..9c9303ccad9 100644 --- a/test/integration/render-tests/heatmap-weight/default/style.json +++ b/test/integration/render-tests/heatmap-weight/default/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.00593, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-weight/identity-property-function/style.json b/test/integration/render-tests/heatmap-weight/identity-property-function/style.json index 52dd5348db5..c7eea307ffb 100644 --- a/test/integration/render-tests/heatmap-weight/identity-property-function/style.json +++ b/test/integration/render-tests/heatmap-weight/identity-property-function/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.00627, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/integration/render-tests/heatmap-weight/literal/style.json b/test/integration/render-tests/heatmap-weight/literal/style.json index 956e52a58ba..71b6af366ea 100644 --- a/test/integration/render-tests/heatmap-weight/literal/style.json +++ b/test/integration/render-tests/heatmap-weight/literal/style.json @@ -4,6 +4,7 @@ "test": { "height": 128, "width": 256, + "allowed": 0.0027, "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform." } }, diff --git a/test/util/html_generator.js b/test/util/html_generator.js index 3ec50436626..b128eda02ad 100644 --- a/test/util/html_generator.js +++ b/test/util/html_generator.js @@ -16,7 +16,7 @@ const generateResultHTML = template(` <% } %> <% if (r.expected) { %> - + <% } %> <% if (r.imgDiff) { %> @@ -81,7 +81,7 @@ const counterDom = { let resultsContainer; -export function setupHTML() { +export function setupHTML(options = {}) { // Add CSS to the page const style = document.createElement('style'); document.head.appendChild(style); @@ -90,6 +90,10 @@ export function setupHTML() { //Create a container to hold test stats const statsContainer = document.createElement('div'); + const webgl2Container = document.createElement('div'); + webgl2Container.innerHTML = options.useWebGL2 ? 'WebGL2 is enabled.' : 'WebGL2 is disabled.'; + statsContainer.appendChild(webgl2Container); + const failedTestContainer = document.createElement('h1'); failedTestContainer.style.color = 'red'; counterDom.failed = document.createElement('span');