Use WebGL2 in render tests by default (#12701)

* Use WebGL2 in render tests by default * Increase the `allowed` value for the heatmap render tests * Add WebGL1 Firefox render test * Log WebGL usage and expected image path
mapbox · May 15, 2023 · e078bf1 · e078bf1
1 parent ab42dec
commit e078bf1
Show file tree

Hide file tree

Showing 17 changed files with 78 additions and 17 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -121,7 +121,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test-render-linux-firefox-dev:
+      - test-render-linux-chrome-webgl1-dev:
           requires:
             - prepare-linux
           filters:
@@ -139,6 +139,18 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - test-render-linux-firefox-dev:
+          requires:
+            - prepare-linux
+          filters:
+            tags:
+              only: /.*/
+      - test-render-linux-firefox-webgl1-dev:
+          requires:
+            - prepare-linux
+          filters:
+            tags:
+              only: /.*/
       - prepare-mac:
           filters:
             tags:
@@ -149,7 +161,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test-render-mac-chrome-webgl2-dev:
+      - test-render-mac-chrome-webgl1-dev:
           requires:
             - prepare-mac
           filters:
@@ -161,7 +173,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test-render-mac-safari-webgl2-dev:
+      - test-render-mac-safari-webgl1-dev:
           requires:
             - prepare-mac
           filters:
@@ -423,6 +435,21 @@ jobs:
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
+  test-render-linux-chrome-webgl1-dev:
+    <<: *linux-defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+      - browser-tools/install-chrome
+      - run:
+          name: Running tests in parallel
+          command: |
+            USE_WEBGL2=false yarn run test-render
+      - store_test_results:
+          path: test/integration/render-tests
+      - store_artifacts:
+          path: "test/integration/render-tests/index.html"
+
   test-render-linux-chrome-prod:
     <<: *linux-defaults
     steps:
@@ -459,6 +486,18 @@ jobs:
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
+  test-render-linux-firefox-webgl1-dev:
+    <<: *linux-defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+      - browser-tools/install-firefox
+      - run: USE_WEBGL2=false yarn run test-render-firefox
+      - store_test_results:
+          path: test/integration/render-tests
+      - store_artifacts:
+          path: "test/integration/render-tests/index.html"
+
   prepare-mac:
     <<: *mac-defaults
     steps:
@@ -497,7 +536,7 @@ jobs:
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
-  test-render-mac-chrome-webgl2-dev:
+  test-render-mac-chrome-webgl1-dev:
     <<: *mac-defaults
     parallelism: 3
     steps:
@@ -508,7 +547,7 @@ jobs:
           name: Creating test list
           command: |
             circleci tests glob "test/integration/render-tests/**/*.json" | circleci tests split --split-by=timings > tests-to-run.txt
-      - run: USE_WEBGL2=true yarn run test-render
+      - run: USE_WEBGL2=false yarn run test-render
       - store_test_results:
           path: test/integration/render-tests
       - store_artifacts:
@@ -526,13 +565,13 @@ jobs:
       - store_artifacts:
           path: "test/integration/render-tests/index.html"
 
-  test-render-mac-safari-webgl2-dev:
+  test-render-mac-safari-webgl1-dev:
     <<: *mac-defaults
     resource_class: macos.m1.large.gen1
     steps:
       - attach_workspace:
           at: ~/
-      - run: USE_WEBGL2=true yarn run test-render-safari
+      - run: USE_WEBGL2=false yarn run test-render-safari
       - store_test_results:
           path: test/integration/render-tests
       - store_artifacts:

diff --git a/test/integration/lib/render.js b/test/integration/lib/render.js
@@ -20,6 +20,8 @@ import {vec3, vec4} from 'gl-matrix';
 
 const browserWriteFile = new Worker('../util/browser_write_file.js');
 
+const useWebGL2 = process.env.USE_WEBGL2 ? process.env.USE_WEBGL2 !== 'false' : true;
+
 // We are self-hosting test files.
 config.REQUIRE_ACCESS_TOKEN = false;
 window._suiteName = 'render-tests';
@@ -43,7 +45,7 @@ fakeCanvasContainer.style.top = '10px';
 fakeCanvasContainer.style.left = '10px';
 document.body.appendChild(fakeCanvasContainer);
 
-setupHTML();
+setupHTML({useWebGL2});
 
 const {canvas: expectedCanvas, ctx: expectedCtx} = createCanvas();
 const {canvas: diffCanvas, ctx: diffCtx} = createCanvas();
@@ -129,13 +131,12 @@ function parseStyle(currentFixture) {
     return style;
 }
 
-function parseOptions(currentFixture, style, useWebGL2) {
+function parseOptions(currentFixture, style) {
     const options = {
         width: 512,
         height: 512,
         pixelRatio: 1,
         allowed: 0.00015,
-        useWebGL2,
         ...((style.metadata && style.metadata.test) || {})
     };
 
@@ -192,7 +193,7 @@ async function renderMap(style, options) {
         attributionControl: false,
         preserveDrawingBuffer: true,
         axonometric: options.axonometric || false,
-        useWebGL2: options.useWebGL2 || false,
+        useWebGL2,
         skew: options.skew || [0, 0],
         fadeDuration: options.fadeDuration || 0,
         optimizeForTerrain: options.optimizeForTerrain || false,
@@ -282,6 +283,7 @@ function getActualImageDataURL(actualImageData, map, {w, h}, options) {
 
 function calculateDiff(actualImageData, expectedImages, {w, h}) {
     // 2. draw expected.png into a canvas and extract ImageData
+    let minImageSrc;
     let minDiffImage;
     let minExpectedCanvas;
     let minDiff = Infinity;
@@ -298,10 +300,11 @@ function calculateDiff(actualImageData, expectedImages, {w, h}) {
             minDiff = currentDiff;
             minDiffImage = diffImage;
             minExpectedCanvas = expectedCanvas;
+            minImageSrc = expectedImages[i].src;
         }
     }
 
-    return {minDiff, minDiffImage, minExpectedCanvas};
+    return {minDiff, minDiffImage, minExpectedCanvas, minImageSrc};
 }
 
 async function getActualImage(style, options) {
@@ -323,7 +326,7 @@ async function runTest(t) {
         const expectedImages = await getExpectedImages(currentTestName, currentFixture);
 
         const style = parseStyle(currentFixture);
-        const options = parseOptions(currentFixture, style, process.env.USE_WEBGL2 && process.env.USE_WEBGL2 === 'true');
+        const options = parseOptions(currentFixture, style);
         const {actualImageData, w, h} = await getActualImage(style, options);
 
         if (process.env.UPDATE) {
@@ -335,7 +338,7 @@ async function runTest(t) {
             return;
         }
 
-        const {minDiff, minDiffImage, minExpectedCanvas} = calculateDiff(actualImageData, expectedImages, {w, h});
+        const {minDiff, minDiffImage, minExpectedCanvas, minImageSrc} = calculateDiff(actualImageData, expectedImages, {w, h});
         const pass = minDiff <= options.allowed;
         const testMetaData = {
             name: currentTestName,
@@ -375,6 +378,7 @@ async function runTest(t) {
             // 7. pass image paths to testMetaData so the UI can render them
             testMetaData.actual = actual;
             testMetaData.expected = minExpectedCanvas.toDataURL();
+            testMetaData.expectedPath = minImageSrc;
             testMetaData.imgDiff = imgDiff;
         }
 

diff --git a/test/integration/render-tests/fog/2d/heatmap/style.json b/test/integration/render-tests/fog/2d/heatmap/style.json
@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 256,
-      "width": 256
+      "width": 256,
+      "allowed": 0.00265
     }
   },
   "center": [

diff --git a/test/integration/render-tests/heatmap-color/default/style.json b/test/integration/render-tests/heatmap-color/default/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 64,
       "width": 256,
+      "allowed": 0.0073,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-color/expression/style.json b/test/integration/render-tests/heatmap-color/expression/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 64,
       "width": 256,
+      "allowed": 0.0073,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-intensity/default/style.json b/test/integration/render-tests/heatmap-intensity/default/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.00593,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-intensity/function/style.json b/test/integration/render-tests/heatmap-intensity/function/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.00263,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-intensity/literal/style.json b/test/integration/render-tests/heatmap-intensity/literal/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.0027,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-opacity/default/style.json b/test/integration/render-tests/heatmap-opacity/default/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 32,
       "width": 128,
+      "allowed": 0.0044,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-radius/default/style.json b/test/integration/render-tests/heatmap-radius/default/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.00593,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-radius/function/style.json b/test/integration/render-tests/heatmap-radius/function/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.01429,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-radius/literal/style.json b/test/integration/render-tests/heatmap-radius/literal/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.01164,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-radius/pitch30/style.json b/test/integration/render-tests/heatmap-radius/pitch30/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.02394,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-weight/default/style.json b/test/integration/render-tests/heatmap-weight/default/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.00593,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-weight/identity-property-function/style.json b/test/integration/render-tests/heatmap-weight/identity-property-function/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.00627,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/integration/render-tests/heatmap-weight/literal/style.json b/test/integration/render-tests/heatmap-weight/literal/style.json
@@ -4,6 +4,7 @@
     "test": {
       "height": 128,
       "width": 256,
+      "allowed": 0.0027,
       "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },

diff --git a/test/util/html_generator.js b/test/util/html_generator.js
@@ -16,7 +16,7 @@ const generateResultHTML = template(`
           <img title="actual" src="<%- r.actual %>">
       <% } %>
       <% if (r.expected) { %>
-          <img title="expected" src="<%- r.expected %>">
+          <img title="expected <%- r.expectedPath %>" src="<%- r.expected %>">
       <% } %>
       <% if (r.imgDiff) { %>
           <img title="diff" src="<%- r.imgDiff %>">
@@ -81,7 +81,7 @@ const counterDom = {
 
 let resultsContainer;
 
-export function setupHTML() {
+export function setupHTML(options = {}) {
     // Add CSS to the page
     const style = document.createElement('style');
     document.head.appendChild(style);
@@ -90,6 +90,10 @@ export function setupHTML() {
     //Create a container to hold test stats
     const statsContainer = document.createElement('div');
 
+    const webgl2Container = document.createElement('div');
+    webgl2Container.innerHTML = options.useWebGL2 ? 'WebGL2 is enabled.' : 'WebGL2 is disabled.';
+    statsContainer.appendChild(webgl2Container);
+
     const failedTestContainer = document.createElement('h1');
     failedTestContainer.style.color = 'red';
     counterDom.failed = document.createElement('span');