diff --git a/README.md b/README.md index 3e432a8d..4cde5157 100644 --- a/README.md +++ b/README.md @@ -178,3 +178,28 @@ differences. If you want to update the reference screenshot for a given example, you can grab those from the build artifacts as well and commit them to your branch. + +### Testing Locally + +Testing locally is possible, however pixel perfect results will differ from +those on the CIs due to discrepencies in hardware, and driver (we use llvmpipe) +versions. + +On linux, it is possible to force to force the usage of LLVMPIPE in the test suite +and compare the generated results of screenshots. Beware, the results on your machine +may differ to those on the CI. We always include the CI screenshots in the test suite +to improve the repeatability of the tests. + +If you have access to a linux machine with llvmpipe installed, you may run the +example pixel comparison testing by setting the WGPUPY_WGPU_ADAPTER_NAME +environment variable appropriately. For example + + +``` +WGPUPY_WGPU_ADAPTER_NAME=llvmpipe pytest -v examples/ +``` + +The `WGPUPY_WGPU_ADAPTER_NAME` variable is modeled after the +https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables +and should only be used for testing the wgpu-py library itself. +It is not part of the supported wgpu-py interface. diff --git a/examples/tests/test_examples.py b/examples/tests/test_examples.py index a9b9b1d1..955df62e 100644 --- a/examples/tests/test_examples.py +++ b/examples/tests/test_examples.py @@ -94,7 +94,10 @@ def unload_module(): # the first part of the test everywhere else; ensuring that examples # can at least import, run and render something if not is_lavapipe: - pytest.skip("screenshot comparisons are only done when using lavapipe") + pytest.skip( + "screenshot comparisons are only done when using lavapipe. " + "Rerun your tests with WGPUPY_WGPU_ADAPTER_NAME=llvmpipe" + ) # regenerate screenshot if requested screenshots_dir.mkdir(exist_ok=True) @@ -108,36 +111,62 @@ def unload_module(): ), "found # test_example = true but no reference screenshot available" stored_img = imageio.imread(screenshot_path) # assert similarity - is_similar = np.allclose(img, stored_img, atol=1) - update_diffs(module, is_similar, img, stored_img) - assert is_similar, ( - f"rendered image for example {module} changed, see " - f"the {diffs_dir.relative_to(ROOT).as_posix()} folder" - " for visual diffs (you can download this folder from" - " CI build artifacts as well)" - ) + atol = 1 + try: + np.testing.assert_allclose(img, stored_img, atol=atol) + is_similar = True + except Exception as e: + is_similar = False + raise AssertionError( + f"rendered image for example {module_name} changed, see " + f"the {diffs_dir.relative_to(ROOT).as_posix()} folder" + " for visual diffs (you can download this folder from" + " CI build artifacts as well)" + ) from e + finally: + update_diffs(module_name, is_similar, img, stored_img, atol=atol) -def update_diffs(module, is_similar, img, stored_img): +def update_diffs(module, is_similar, img, stored_img, *, atol): diffs_dir.mkdir(exist_ok=True) + + if is_similar: + for path in [ + # Keep filename in sync with the ones generated below + diffs_dir / f"{module}-rgb.png", + diffs_dir / f"{module}-alpha.png", + diffs_dir / f"{module}-rgb-above_atol.png", + diffs_dir / f"{module}-alpha-above_atol.png", + diffs_dir / f"{module}.png", + ]: + if path.exists(): + path.unlink() + return + # cast to float32 to avoid overflow # compute absolute per-pixel difference diffs_rgba = np.abs(stored_img.astype("f4") - img) + + diffs_rgba_above_atol = diffs_rgba.copy() + diffs_rgba_above_atol[diffs_rgba <= atol] = 0 + # magnify small values, making it easier to spot small errors diffs_rgba = ((diffs_rgba / 255) ** 0.25) * 255 # cast back to uint8 diffs_rgba = diffs_rgba.astype("u1") - # split into an rgb and an alpha diff - diffs = { - diffs_dir / f"{module}-rgb.png": diffs_rgba[..., :3], - diffs_dir / f"{module}-alpha.png": diffs_rgba[..., 3], - } - - for path, diff in diffs.items(): - if not is_similar: - imageio.imwrite(path, diff) - elif path.exists(): - path.unlink() + + diffs_rgba_above_atol = ((diffs_rgba_above_atol / 255) ** 0.25) * 255 + diffs_rgba_above_atol = diffs_rgba_above_atol.astype("u1") + # And highlight differences that are above the atol + imageio.imwrite(diffs_dir / f"{module}-rgb.png", diffs_rgba[..., :3]) + imageio.imwrite(diffs_dir / f"{module}-alpha.png", diffs_rgba[..., 3]) + imageio.imwrite( + diffs_dir / f"{module}-rgb-above_atol.png", diffs_rgba_above_atol[..., :3] + ) + imageio.imwrite( + diffs_dir / f"{module}-alpha-above_atol.png", diffs_rgba_above_atol[..., 3] + ) + imageio.imwrite(diffs_dir / f"{module}.png", img) @pytest.mark.parametrize("module", examples_to_run) diff --git a/wgpu/backends/wgpu_native/_api.py b/wgpu/backends/wgpu_native/_api.py index 4b2177ba..dffd010c 100644 --- a/wgpu/backends/wgpu_native/_api.py +++ b/wgpu/backends/wgpu_native/_api.py @@ -404,6 +404,27 @@ async def request_adapter_async( def _request_adapter( self, *, power_preference=None, force_fallback_adapter=False, canvas=None ): + # Similar to https://github.com/gfx-rs/wgpu?tab=readme-ov-file#environment-variables + # It seems that the environment variables are only respected in their + # testing environments maybe???? + # In Dec 2024 we couldn't get the use of their environment variables to work + # This should only be used in testing environments and API users + # should beware + # We chose the variable name WGPUPY_WGPU_ADAPTER_NAME instead WGPU_ADAPTER_NAME + # to avoid a clash + if adapter_name := os.getenv(("WGPUPY_WGPU_ADAPTER_NAME")): + adapters = self._enumerate_adapters() + adapters_llvm = [a for a in adapters if adapter_name in a.summary] + if not adapters_llvm: + raise ValueError(f"Adapter with name '{adapter_name}' not found.") + awaitable = WgpuAwaitable( + "llvm adapter", + callback=lambda: (), + finalizer=lambda x: x, + ) + awaitable.set_result(adapters_llvm[0]) + + return awaitable # ----- Surface ID # Get surface id that the adapter must be compatible with. If we