Improve error reporting with screenshots and provide a method

to test locally

README.md

@@ -178,3 +178,16 @@ differences.
 
 If you want to update the reference screenshot for a given example, you can grab
 those from the build artifacts as well and commit them to your branch.
+
+### Testing Locally
+
+Testing locally is possible, however pixel perfect results will differ from
+those on the CIs due to discrepencies in hardware, and driver (we use llvmpipe)
+versions.
+
+If you want to force the usage of LLVMPIPE to speed up local testing you
+may do so with the WGPUPY_WGPU_ADAPTER_NAME environment variable
+
+```
+WGPUPY_WGPU_ADAPTER_NAME=llvmpipe pytest -v examples/
+```

examples/tests/test_examples.py

@@ -94,7 +94,10 @@ def unload_module():
     # the first part of the test everywhere else; ensuring that examples
     # can at least import, run and render something
     if not is_lavapipe:
-        pytest.skip("screenshot comparisons are only done when using lavapipe")
+        pytest.skip(
+            "screenshot comparisons are only done when using lavapipe. "
+            "Rerun your tests with WGPUPY_WGPU_ADAPTER_NAME=llvmpipe"
+        )
 
     # regenerate screenshot if requested
     screenshots_dir.mkdir(exist_ok=True)
@@ -108,36 +111,62 @@ def unload_module():
     ), "found # test_example = true but no reference screenshot available"
     stored_img = imageio.imread(screenshot_path)
     # assert similarity
-    is_similar = np.allclose(img, stored_img, atol=1)
-    update_diffs(module, is_similar, img, stored_img)
-    assert is_similar, (
-        f"rendered image for example {module} changed, see "
-        f"the {diffs_dir.relative_to(ROOT).as_posix()} folder"
-        " for visual diffs (you can download this folder from"
-        " CI build artifacts as well)"
-    )
+    atol = 1
+    try:
+        np.testing.assert_allclose(img, stored_img, atol=atol)
+        is_similar = True
+    except Exception as e:
+        is_similar = False
+        raise AssertionError(
+            f"rendered image for example {module_name} changed, see "
+            f"the {diffs_dir.relative_to(ROOT).as_posix()} folder"
+            " for visual diffs (you can download this folder from"
+            " CI build artifacts as well)"
+        ) from e
+    finally:
+        update_diffs(module_name, is_similar, img, stored_img, atol=atol)
 
 
-def update_diffs(module, is_similar, img, stored_img):
+def update_diffs(module, is_similar, img, stored_img, *, atol):
     diffs_dir.mkdir(exist_ok=True)
+
+    if is_similar:
+        for path in [
+            # Keep filename in sync with the ones generated below
+            diffs_dir / f"{module}-rgb.png",
+            diffs_dir / f"{module}-alpha.png",
+            diffs_dir / f"{module}-rgb-above_atol.png",
+            diffs_dir / f"{module}-alpha-above_atol.png",
+            diffs_dir / f"{module}.png",
+        ]:
+            if path.exists():
+                path.unlink()
+        return
+
     # cast to float32 to avoid overflow
     # compute absolute per-pixel difference
     diffs_rgba = np.abs(stored_img.astype("f4") - img)
+
+    diffs_rgba_above_atol = diffs_rgba.copy()
+    diffs_rgba_above_atol[diffs_rgba <= atol] = 0
+
     # magnify small values, making it easier to spot small errors
     diffs_rgba = ((diffs_rgba / 255) ** 0.25) * 255
     # cast back to uint8
     diffs_rgba = diffs_rgba.astype("u1")
-    # split into an rgb and an alpha diff
-    diffs = {
-        diffs_dir / f"{module}-rgb.png": diffs_rgba[..., :3],
-        diffs_dir / f"{module}-alpha.png": diffs_rgba[..., 3],
-    }
-
-    for path, diff in diffs.items():
-        if not is_similar:
-            imageio.imwrite(path, diff)
-        elif path.exists():
-            path.unlink()
+
+    diffs_rgba_above_atol = ((diffs_rgba_above_atol / 255) ** 0.25) * 255
+    diffs_rgba_above_atol = diffs_rgba_above_atol.astype("u1")
+    # And highlight differences that are above the atol
+    imageio.imwrite(diffs_dir / f"{module}-rgb.png", diffs_rgba[..., :3])
+    imageio.imwrite(diffs_dir / f"{module}-alpha.png", diffs_rgba[..., 3])
+    imageio.imwrite(
+        diffs_dir / f"{module}-rgb-above_atol.png", diffs_rgba_above_atol[..., :3]
+    )
+    imageio.imwrite(
+        diffs_dir / f"{module}-alpha-above_atol.png", diffs_rgba_above_atol[..., 3]
+    )
+    imageio.imwrite(diffs_dir / f"{module}.png", img)
 
 
 @pytest.mark.parametrize("module", examples_to_run)

wgpu/backends/wgpu_native/_api.py

@@ -367,6 +367,21 @@ def request_adapter_sync(
         This is the implementation based on wgpu-native.
         """
         check_can_use_sync_variants()
+        # Similar to https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables
+        # It seems that the environment variables are only respected in their
+        # testing environments maybe????
+        # In Dec 2024 we couldn't get the use of their environment variables to work
+        # This should only be used in testing environments and API users
+        # should beware
+        # We chose the variable name WGPUPY_WGPU_ADAPTER_NAME instead WGPU_ADAPTER_NAME
+        # to avoid a clash
+        if adapter_name := os.getenv(("WGPUPY_WGPU_ADAPTER_NAME")):
+            adapters = self.enumerate_adapters_sync()
+            adapters_llvm = [a for a in adapters if adapter_name in a.summary]
+            if not adapters_llvm:
+                raise ValueError(f"Adapter with name '{adapter_name}' not found.")
+            return adapters_llvm[0]
+
         awaitable = self._request_adapter(
             power_preference=power_preference,
             force_fallback_adapter=force_fallback_adapter,
@@ -394,6 +409,23 @@ async def request_adapter_async(
             canvas : The canvas that the adapter should be able to render to. This can typically
                  be left to None. If given, the object must implement ``WgpuCanvasInterface``.
         """
+        # Similar to https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables
+        # It seems that the environment variables are only respected in their
+        # testing environments maybe????
+        # In Dec 2024 we couldn't get the use of their environment variables to work
+        # This should only be used in testing environments and API users
+        # should beware
+        # We chose the variable name WGPUPY_WGPU_ADAPTER_NAME instead WGPU_ADAPTER_NAME
+        # to avoid a clash
+        if adapter_name := os.getenv(("WGPUPY_WGPU_ADAPTER_NAME")):
+            # Is this correct for async??? I know nothing of async...
+            awaitable = self.enumerate_adapters_async()
+            adapters = await awaitable
+            adapters_llvm = [a for a in adapters if adapter_name in a.summary]
+            if not adapters_llvm:
+                raise ValueError(f"Adapter with name '{adapter_name}' not found.")
+            return adapters_llvm[0]
+
         awaitable = self._request_adapter(
             power_preference=power_preference,
             force_fallback_adapter=force_fallback_adapter,