Improve state of the art for screenshot testing (#555)

* Improve error reporting with screenshots and provide a method
to test locally

* improve readme.

* Break everything because i don't understand async

* run ruff format...

* spelling

* minimize changes

README.md

@@ -178,3 +178,28 @@ differences.
 
 If you want to update the reference screenshot for a given example, you can grab
 those from the build artifacts as well and commit them to your branch.
+
+### Testing Locally
+
+Testing locally is possible, however pixel perfect results will differ from
+those on the CIs due to discrepencies in hardware, and driver (we use llvmpipe)
+versions.
+
+On linux, it is possible to force to force the usage of LLVMPIPE in the test suite
+and compare the generated results of screenshots. Beware, the results on your machine
+may differ to those on the CI. We always include the CI screenshots in the test suite
+to improve the repeatability of the tests.
+
+If you have access to a linux machine with llvmpipe installed, you may run the
+example pixel comparison testing by setting the WGPUPY_WGPU_ADAPTER_NAME
+environment variable appropriately. For example
+
+
+```
+WGPUPY_WGPU_ADAPTER_NAME=llvmpipe pytest -v examples/
+```
+
+The `WGPUPY_WGPU_ADAPTER_NAME` variable is modeled after the
+https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables
+and should only be used for testing the wgpu-py library itself.
+It is not part of the supported wgpu-py interface.

examples/tests/test_examples.py

@@ -94,7 +94,10 @@ def unload_module():
     # the first part of the test everywhere else; ensuring that examples
     # can at least import, run and render something
     if not is_lavapipe:
-        pytest.skip("screenshot comparisons are only done when using lavapipe")
+        pytest.skip(
+            "screenshot comparisons are only done when using lavapipe. "
+            "Rerun your tests with WGPUPY_WGPU_ADAPTER_NAME=llvmpipe"
+        )
 
     # regenerate screenshot if requested
     screenshots_dir.mkdir(exist_ok=True)
@@ -108,36 +111,62 @@ def unload_module():
     ), "found # test_example = true but no reference screenshot available"
     stored_img = imageio.imread(screenshot_path)
     # assert similarity
-    is_similar = np.allclose(img, stored_img, atol=1)
-    update_diffs(module, is_similar, img, stored_img)
-    assert is_similar, (
-        f"rendered image for example {module} changed, see "
-        f"the {diffs_dir.relative_to(ROOT).as_posix()} folder"
-        " for visual diffs (you can download this folder from"
-        " CI build artifacts as well)"
-    )
+    atol = 1
+    try:
+        np.testing.assert_allclose(img, stored_img, atol=atol)
+        is_similar = True
+    except Exception as e:
+        is_similar = False
+        raise AssertionError(
+            f"rendered image for example {module_name} changed, see "
+            f"the {diffs_dir.relative_to(ROOT).as_posix()} folder"
+            " for visual diffs (you can download this folder from"
+            " CI build artifacts as well)"
+        ) from e
+    finally:
+        update_diffs(module_name, is_similar, img, stored_img, atol=atol)
 
 
-def update_diffs(module, is_similar, img, stored_img):
+def update_diffs(module, is_similar, img, stored_img, *, atol):
     diffs_dir.mkdir(exist_ok=True)
+
+    if is_similar:
+        for path in [
+            # Keep filename in sync with the ones generated below
+            diffs_dir / f"{module}-rgb.png",
+            diffs_dir / f"{module}-alpha.png",
+            diffs_dir / f"{module}-rgb-above_atol.png",
+            diffs_dir / f"{module}-alpha-above_atol.png",
+            diffs_dir / f"{module}.png",
+        ]:
+            if path.exists():
+                path.unlink()
+        return
+
     # cast to float32 to avoid overflow
     # compute absolute per-pixel difference
     diffs_rgba = np.abs(stored_img.astype("f4") - img)
+
+    diffs_rgba_above_atol = diffs_rgba.copy()
+    diffs_rgba_above_atol[diffs_rgba <= atol] = 0
+
     # magnify small values, making it easier to spot small errors
     diffs_rgba = ((diffs_rgba / 255) ** 0.25) * 255
     # cast back to uint8
     diffs_rgba = diffs_rgba.astype("u1")
-    # split into an rgb and an alpha diff
-    diffs = {
-        diffs_dir / f"{module}-rgb.png": diffs_rgba[..., :3],
-        diffs_dir / f"{module}-alpha.png": diffs_rgba[..., 3],
-    }
-
-    for path, diff in diffs.items():
-        if not is_similar:
-            imageio.imwrite(path, diff)
-        elif path.exists():
-            path.unlink()
+
+    diffs_rgba_above_atol = ((diffs_rgba_above_atol / 255) ** 0.25) * 255
+    diffs_rgba_above_atol = diffs_rgba_above_atol.astype("u1")
+    # And highlight differences that are above the atol
+    imageio.imwrite(diffs_dir / f"{module}-rgb.png", diffs_rgba[..., :3])
+    imageio.imwrite(diffs_dir / f"{module}-alpha.png", diffs_rgba[..., 3])
+    imageio.imwrite(
+        diffs_dir / f"{module}-rgb-above_atol.png", diffs_rgba_above_atol[..., :3]
+    )
+    imageio.imwrite(
+        diffs_dir / f"{module}-alpha-above_atol.png", diffs_rgba_above_atol[..., 3]
+    )
+    imageio.imwrite(diffs_dir / f"{module}.png", img)
 
 
 @pytest.mark.parametrize("module", examples_to_run)

wgpu/backends/wgpu_native/_api.py

@@ -404,6 +404,27 @@ async def request_adapter_async(
     def _request_adapter(
         self, *, power_preference=None, force_fallback_adapter=False, canvas=None
     ):
+        # Similar to https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables
+        # It seems that the environment variables are only respected in their
+        # testing environments maybe????
+        # In Dec 2024 we couldn't get the use of their environment variables to work
+        # This should only be used in testing environments and API users
+        # should beware
+        # We chose the variable name WGPUPY_WGPU_ADAPTER_NAME instead WGPU_ADAPTER_NAME
+        # to avoid a clash
+        if adapter_name := os.getenv(("WGPUPY_WGPU_ADAPTER_NAME")):
+            adapters = self._enumerate_adapters()
+            adapters_llvm = [a for a in adapters if adapter_name in a.summary]
+            if not adapters_llvm:
+                raise ValueError(f"Adapter with name '{adapter_name}' not found.")
+            awaitable = WgpuAwaitable(
+                "llvm adapter",
+                callback=lambda: (),
+                finalizer=lambda x: x,
+            )
+            awaitable.set_result(adapters_llvm[0])
+
+            return awaitable
         # ----- Surface ID
 
         # Get surface id that the adapter must be compatible with. If we