Migrate to nv-flip for image comparison (#3830)

Co-authored-by: Connor Fitzgerald <[email protected]>

.github/workflows/ci.yml

@@ -10,6 +10,7 @@ on:
 env:
   CARGO_INCREMENTAL: false
   CARGO_TERM_COLOR: always
+  RUST_LOG: info
   RUST_BACKTRACE: full
   MSRV: 1.65
   PKG_CONFIG_ALLOW_CROSS: 1 # allow android to work
@@ -112,10 +113,23 @@ jobs:
         with:
           key: clippy-${{ matrix.target }}-${{ matrix.kind }}-${{ env.CACHE_SUFFIX }}
 
+      - name: install aarch64-linux-gnu g++
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        run: |
+          set -e
+
+          sudo apt-get update -y -qq
+
+          sudo apt-get install g++-aarch64-linux-gnu
+
       - name: add android apk to path
         if: matrix.target == 'aarch64-linux-android'
         run: |
-          echo "$ANDROID_HOME/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin" >> $GITHUB_PATH
+          # clang++ will be detected correctly by CC from path
+          echo "$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin" >> $GITHUB_PATH
+
+          # the android sdk doesn't use the conventional name for ar, so explicitly set it.
+          echo "AR_aarch64_linux_android=llvm-ar" >> "$GITHUB_ENV"
 
       - name: check web
         if: matrix.kind == 'web'
@@ -224,7 +238,7 @@ jobs:
           mkdir -p swiftshader
           curl -LsSf https://github.com/gfx-rs/ci-build/releases/latest/download/swiftshader-linux-x86_64.tar.xz | tar -xf - -C swiftshader
 
-          echo "LD_LIBRARY_PATH=$PWD/swiftshader" >> $GITHUB_ENV
+          echo "VK_ICD_FILENAMES=$PWD/swiftshader/vk_swiftshader_icd.json" >> $GITHUB_ENV
 
       - name: install llvmpipe, vulkan sdk
         if: matrix.os == 'ubuntu-22.04'
@@ -272,6 +286,13 @@ jobs:
             WGPU_BACKEND=$backend cargo llvm-cov --no-cfg-coverage nextest -p wgpu -p wgpu-types -p wgpu-hal -p wgpu-core -p player --no-fail-fast --no-report
           done
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: comparison-images
+          path: |
+            **/*-actual.png
+            **/*-difference.png
+
       - name: generate coverage report
         shell: bash
         run: |

Cargo.toml

@@ -65,6 +65,7 @@ libloading = ">=0.7,<0.9"
 libc = "0.2"
 log = "0.4"
 nanorand = { version = "0.7", default-features = false }
+nv-flip = "0.1"
 num-traits = { version = "0.2" }
 # Opt out of noise's "default-features" to avoid "image" feature as a dependency count optimization.
 # This will not be required in the next release since it has been removed from the default feature in https://github.com/Razaekel/noise-rs/commit/1af9e1522236b2c584fb9a02150c9c67a5e6bb04#diff-2e9d962a08321605940b5a657135052fbcef87b5e360662bb527c96d9a615542
@@ -149,3 +150,7 @@ wgpu-types = { path = "./wgpu-types" }
 #web-sys = { path = "../wasm-bindgen/crates/web-sys" }
 #js-sys = { path = "../wasm-bindgen/crates/js-sys" }
 #wasm-bindgen = { path = "../wasm-bindgen" }
+
+# Speed up image comparison even in debug builds
+[profile.dev.package."nv-flip-sys"]
+opt-level = 3

wgpu-types/src/lib.rs

@@ -111,6 +111,21 @@ pub enum Backend {
     BrowserWebGpu = 6,
 }
 
+impl Backend {
+    /// Returns the string name of the backend.
+    pub fn to_str(self) -> &'static str {
+        match self {
+            Backend::Empty => "empty",
+            Backend::Vulkan => "vulkan",
+            Backend::Metal => "metal",
+            Backend::Dx12 => "dx12",
+            Backend::Dx11 => "dx11",
+            Backend::Gl => "gl",
+            Backend::BrowserWebGpu => "webgpu",
+        }
+    }
+}
+
 /// Power Preference when choosing a physical adapter.
 ///
 /// Corresponds to [WebGPU `GPUPowerPreference`](

wgpu/Cargo.toml

@@ -178,6 +178,7 @@ winit.workspace = true # for "halmark" example
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 async-executor.workspace = true
+nv-flip.workspace = true
 
 [dependencies.naga]
 workspace = true

wgpu/examples/boids/main.rs

@@ -335,14 +335,14 @@ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
 #[wasm_bindgen_test::wasm_bindgen_test]
 fn boids() {
     framework::test::<Example>(framework::FrameworkRefTest {
+        // Generated on 1080ti on Vk/Windows
         image_path: "/examples/boids/screenshot.png",
         width: 1024,
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default()
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults()),
-        tolerance: 0,
-        max_outliers: 2500, // Currently bounded by WARP
+        comparisons: &[framework::ComparisonType::Mean(0.005)],
     });
 }

wgpu/examples/bunnymark/main.rs

@@ -369,7 +369,13 @@ fn bunnymark() {
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 10,
-        max_outliers: 53, // Bounded by WARP
+        // We're looking for very small differences, so look in the high percentiles.
+        comparisons: &[
+            framework::ComparisonType::Mean(0.05),
+            framework::ComparisonType::Percentile {
+                percentile: 0.95,
+                threshold: 0.05,
+            },
+        ],
     });
 }

wgpu/examples/conservative-raster/main.rs

@@ -326,7 +326,6 @@ fn conservative_raster() {
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 0,
-        max_outliers: 0,
+        comparisons: &[framework::ComparisonType::Mean(0.0)],
     });
 }

wgpu/examples/cube/main.rs

@@ -413,26 +413,35 @@ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
 #[wasm_bindgen_test::wasm_bindgen_test]
 fn cube() {
     framework::test::<Example>(framework::FrameworkRefTest {
+        // Generated on 1080ti on Vk/Windows
         image_path: "/examples/cube/screenshot.png",
         width: 1024,
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 1,
-        max_outliers: 1225, // Bounded by swiftshader
+        comparisons: &[
+            framework::ComparisonType::Mean(0.04), // Bounded by Intel 630 on Vk/Windows
+        ],
     });
 }
 
 #[test]
 #[wasm_bindgen_test::wasm_bindgen_test]
 fn cube_lines() {
     framework::test::<Example>(framework::FrameworkRefTest {
+        // Generated on 1080ti on Vk/Windows
         image_path: "/examples/cube/screenshot-lines.png",
         width: 1024,
         height: 768,
         optional_features: wgpu::Features::POLYGON_MODE_LINE,
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 2,
-        max_outliers: 1250, // Bounded by swiftshader
+        // We're looking for tiny changes here, so we focus on a spike in the 95th percentile.
+        comparisons: &[
+            framework::ComparisonType::Mean(0.05), // Bounded by Intel 630 on Vk/Windows
+            framework::ComparisonType::Percentile {
+                percentile: 0.95,
+                threshold: 0.36,
+            }, // Bounded by 1080ti on DX12
+        ],
     });
 }

wgpu/examples/framework.rs

@@ -494,15 +494,18 @@ pub fn parse_url_query_string<'a>(query: &'a str, search_key: &str) -> Option<&'
     None
 }
 
+#[cfg(test)]
+pub use test_common::image::ComparisonType;
+
 #[cfg(test)]
 pub struct FrameworkRefTest {
     pub image_path: &'static str,
     pub width: u32,
     pub height: u32,
     pub optional_features: wgpu::Features,
     pub base_test_parameters: test_common::TestParameters,
-    pub tolerance: u8,
-    pub max_outliers: usize,
+    /// Comparisons against FLIP statistics that determine if the test passes or fails.
+    pub comparisons: &'static [ComparisonType],
 }
 
 #[cfg(test)]
@@ -616,11 +619,11 @@ pub fn test<E: Example>(mut params: FrameworkRefTest) {
 
             test_common::image::compare_image_output(
                 env!("CARGO_MANIFEST_DIR").to_string() + params.image_path,
+                ctx.adapter_info.backend,
                 params.width,
                 params.height,
                 &bytes,
-                params.tolerance,
-                params.max_outliers,
+                params.comparisons,
             );
         },
     );

wgpu/examples/mipmap/main.rs

@@ -500,7 +500,6 @@ fn mipmap() {
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default()
             .backend_failure(wgpu::Backends::GL),
-        tolerance: 50,
-        max_outliers: 5000, // Mipmap sampling is highly variant between impls. This is currently bounded by lavapipe
+        comparisons: &[framework::ComparisonType::Mean(0.02)],
     });
 }

wgpu/examples/msaa-line/main.rs

@@ -324,10 +324,18 @@ fn msaa_line() {
         image_path: "/examples/msaa-line/screenshot.png",
         width: 1024,
         height: 768,
-        optional_features: wgpu::Features::default()
-            | wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES,
-        base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 64,
-        max_outliers: 1 << 16, // MSAA is comically different between vendors, 32k is a decent limit
+        optional_features: wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES,
+        base_test_parameters: framework::test_common::TestParameters::default()
+            // AMD seems to render nothing on DX12 https://github.com/gfx-rs/wgpu/issues/3838
+            .specific_failure(Some(wgpu::Backends::DX12), Some(0x1002), None, false),
+        // There's a lot of natural variance so we check the weighted median too to differentiate
+        // real failures from variance.
+        comparisons: &[
+            framework::ComparisonType::Mean(0.065),
+            framework::ComparisonType::Percentile {
+                percentile: 0.5,
+                threshold: 0.29,
+            },
+        ],
     });
 }

wgpu/examples/shadow/main.rs

@@ -859,7 +859,6 @@ fn shadow() {
             .specific_failure(Some(wgpu::Backends::VULKAN), None, Some("V3D"), false)
             // llvmpipe versions in CI are flaky: https://github.com/gfx-rs/wgpu/issues/2594
             .specific_failure(Some(wgpu::Backends::VULKAN), None, Some("llvmpipe"), true),
-        tolerance: 2,
-        max_outliers: 1075, // bounded by swiftshader
+        comparisons: &[framework::ComparisonType::Mean(0.02)],
     });
 }

wgpu/examples/skybox/main.rs

@@ -482,8 +482,7 @@ fn skybox() {
             Some("ANGLE"),
             false,
         ),
-        tolerance: 3,
-        max_outliers: 207, // bounded by swiftshader
+        comparisons: &[framework::ComparisonType::Mean(0.015)],
     });
 }
 
@@ -496,8 +495,7 @@ fn skybox_bc1() {
         height: 768,
         optional_features: wgpu::Features::TEXTURE_COMPRESSION_BC,
         base_test_parameters: framework::test_common::TestParameters::default(), // https://bugs.chromium.org/p/angleproject/issues/detail?id=7056
-        tolerance: 5,
-        max_outliers: 191, // Bounded by swiftshader
+        comparisons: &[framework::ComparisonType::Mean(0.02)],
     });
 }
 
@@ -510,8 +508,7 @@ fn skybox_etc2() {
         height: 768,
         optional_features: wgpu::Features::TEXTURE_COMPRESSION_ETC2,
         base_test_parameters: framework::test_common::TestParameters::default(), // https://bugs.chromium.org/p/angleproject/issues/detail?id=7056
-        tolerance: 5,
-        max_outliers: 248, // Bounded by swiftshader
+        comparisons: &[framework::ComparisonType::Mean(0.015)],
     });
 }
 
@@ -524,7 +521,6 @@ fn skybox_astc() {
         height: 768,
         optional_features: wgpu::Features::TEXTURE_COMPRESSION_ASTC,
         base_test_parameters: framework::test_common::TestParameters::default(), // https://bugs.chromium.org/p/angleproject/issues/detail?id=7056
-        tolerance: 5,
-        max_outliers: 300, // Bounded by rp4 on vk
+        comparisons: &[framework::ComparisonType::Mean(0.016)],
     });
 }

wgpu/examples/stencil-triangles/main.rs

@@ -246,7 +246,6 @@ fn stencil_triangles() {
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 1,
-        max_outliers: 0,
+        comparisons: &[framework::ComparisonType::Mean(0.03)],
     });
 }

wgpu/examples/texture-arrays/main.rs

@@ -422,8 +422,7 @@ fn texture_arrays_uniform() {
         height: 768,
         optional_features: wgpu::Features::empty(),
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 0,
-        max_outliers: 0,
+        comparisons: &[framework::ComparisonType::Mean(0.0)],
     });
 }
 
@@ -437,7 +436,6 @@ fn texture_arrays_non_uniform() {
         optional_features:
             wgpu::Features::SAMPLED_TEXTURE_AND_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING,
         base_test_parameters: framework::test_common::TestParameters::default(),
-        tolerance: 0,
-        max_outliers: 0,
+        comparisons: &[framework::ComparisonType::Mean(0.0)],
     });
 }

wgpu/examples/water/main.rs

@@ -832,7 +832,6 @@ fn water() {
         optional_features: wgpu::Features::default(),
         base_test_parameters: framework::test_common::TestParameters::default()
             .downlevel_flags(wgpu::DownlevelFlags::READ_ONLY_DEPTH_STENCIL),
-        tolerance: 5,
-        max_outliers: 1693, // bounded by swiftshader
+        comparisons: &[framework::ComparisonType::Mean(0.01)],
     });
 }