fix(ballot-interpreter): improve vertical streak detection (#5522)

* fix(ballot-interpreter): exclude more ballot edges

When scanning in VxCentralScan sometimes we get wider gray areas outside the ballot paper than we expected. Most of this gray is binarized to black, leading to false positives when detecting streaks. This reduces the likelihood of false positives while preserving detection of streaks within the ballot, including within the timing marks.

* feat(ballot-interpreter): improve vertical streak debug images

Binarizes the debug image to make it clearer what the streak detector was working with.

* test(ballot-interpreter): fix comment ⟷ code mismatch

The comment says to draw on side B, so I updated the code to do that.

* test(ballot-interpreter): add more streak detection tests

Ensures that streaks through timing marks are still detected and semi-wide edge "streaks" do not cause false positives.

libs/ballot-interpreter/src/hmpb-rust/debug.rs

@@ -1,5 +1,6 @@
 #![allow(clippy::too_many_lines)]
 
+use std::ops::Range;
 use std::path::{Path, PathBuf};
 
 use ab_glyph::{FontRef, PxScale};
@@ -106,8 +107,29 @@ pub fn draw_qr_code_debug_image_mut(
 
 pub fn draw_vertical_streaks_debug_image_mut(
     canvas: &mut RgbImage,
-    streaks: &[(PixelUnit, f32, PixelUnit)],
+    threshold: u8,
+    x_range: Range<PixelUnit>,
+    streaks: &[(PixelPosition, UnitIntervalScore, PixelUnit)],
 ) {
+    // binarize the image since that's what the detection algorithm works with
+    for px in canvas.pixels_mut() {
+        if px.0[0] <= threshold {
+            *px = Rgb([0, 0, 0]);
+        } else {
+            *px = Rgb([255, 255, 255]);
+        }
+    }
+
+    // color the area being ignored
+    for x in 0..canvas.width() {
+        if x_range.contains(&x) {
+            continue;
+        }
+        for y in 0..canvas.height() {
+            canvas.put_pixel(x, y, DARK_CYAN);
+        }
+    }
+
     for (i, ((x, percent_black_pixels, longest_white_gap_length), color)) in
         streaks.iter().zip(dark_rainbow()).enumerate()
     {
@@ -116,12 +138,7 @@ pub fn draw_vertical_streaks_debug_image_mut(
         draw_cross_mut(canvas, color, x, y);
         draw_text_with_background_mut(
             canvas,
-            &format!(
-                "x={}, Black: {:.0}%, Gap: {}",
-                x,
-                100.0 * percent_black_pixels,
-                longest_white_gap_length
-            ),
+            &format!("x={x}, Black: {percent_black_pixels}, Gap: {longest_white_gap_length}"),
             x + 5,
             y,
             PxScale::from(20.0),

libs/ballot-interpreter/src/hmpb-rust/image_utils.rs

@@ -7,7 +7,10 @@ use serde::Serialize;
 use types_rs::geometry::{PixelPosition, PixelUnit, Size, SubPixelUnit};
 use types_rs::{election::UnitIntervalValue, geometry::Quadrilateral};
 
-use crate::debug::{self, ImageDebugWriter};
+use crate::{
+    debug::{self, ImageDebugWriter},
+    scoring::UnitIntervalScore,
+};
 
 pub const WHITE: Luma<u8> = Luma([255]);
 pub const BLACK: Luma<u8> = Luma([0]);
@@ -291,9 +294,9 @@ pub fn detect_vertical_streaks(
     threshold: u8,
     debug: &ImageDebugWriter,
 ) -> Vec<PixelPosition> {
-    const PERCENT_BLACK_PIXELS_IN_STREAK: f32 = 0.75;
+    const MIN_STREAK_SCORE: UnitIntervalScore = UnitIntervalScore(0.75);
     const MAX_WHITE_GAP_PIXELS: PixelUnit = 15;
-    const BORDER_COLUMNS_TO_EXCLUDE: PixelUnit = 5;
+    const BORDER_COLUMNS_TO_EXCLUDE: PixelUnit = 20;
 
     // Look at each column of pixels in the image (ignoring
     // BORDER_COLUMNS_TO_EXCLUDE on either side), where a "column" is two pixels
@@ -305,22 +308,22 @@ pub fn detect_vertical_streaks(
     // top to bottom without a gap greater than MAX_WHITE_GAP_PIXELS.
 
     let (width, height) = image.dimensions();
-    let binarized_columns =
-        (BORDER_COLUMNS_TO_EXCLUDE - 1..width - BORDER_COLUMNS_TO_EXCLUDE).map(|x| {
-            let binarized_column = (0..height)
-                .map(|y| {
-                    [image.get_pixel(x, y), image.get_pixel(x + 1, y)]
-                        .iter()
-                        .any(|pixel| pixel[0] <= threshold)
-                })
-                .collect::<Vec<_>>();
-            (x, binarized_column)
-        });
+    let x_range = BORDER_COLUMNS_TO_EXCLUDE - 1..width - BORDER_COLUMNS_TO_EXCLUDE;
+    let binarized_columns = x_range.clone().map(|x| {
+        let binarized_column = (0..height)
+            .map(|y| {
+                [image.get_pixel(x, y), image.get_pixel(x + 1, y)]
+                    .iter()
+                    .any(|pixel| pixel[0] <= threshold)
+            })
+            .collect::<Vec<_>>();
+        (x as PixelPosition, binarized_column)
+    });
 
     let streak_columns = binarized_columns.filter_map(|(x, column)| {
         let num_black_pixels = column.iter().filter(|is_black| **is_black).count();
-        let percent_black_pixels = num_black_pixels as f32 / height as f32;
-        if percent_black_pixels < PERCENT_BLACK_PIXELS_IN_STREAK {
+        let streak_score = UnitIntervalScore(num_black_pixels as f32 / height as f32);
+        if streak_score < MIN_STREAK_SCORE {
             return None;
         }
 
@@ -333,7 +336,7 @@ pub fn detect_vertical_streaks(
             .max()
             .unwrap_or(0);
         if longest_white_gap_length <= MAX_WHITE_GAP_PIXELS {
-            Some((x, percent_black_pixels, longest_white_gap_length))
+            Some((x, streak_score, longest_white_gap_length))
         } else {
             None
         }
@@ -353,13 +356,10 @@ pub fn detect_vertical_streaks(
         .collect::<Vec<_>>();
 
     debug.write("vertical_streaks", |canvas| {
-        debug::draw_vertical_streaks_debug_image_mut(canvas, &streaks);
+        debug::draw_vertical_streaks_debug_image_mut(canvas, threshold, x_range, &streaks);
     });
 
-    streaks
-        .into_iter()
-        .map(|(x, _, _)| x as PixelPosition)
-        .collect::<Vec<_>>()
+    streaks.into_iter().map(|(x, _, _)| x).collect()
 }
 
 /// Calculates the degree to which the given image matches the given template,

libs/ballot-interpreter/src/hmpb-rust/interpret.rs

@@ -856,7 +856,7 @@ mod test {
 
     #[test]
     fn test_vertical_streaks() {
-        let (mut side_a_image, side_b_image, options) =
+        let (mut side_a_image, mut side_b_image, options) =
             load_hmpb_fixture("general-election/letter", 1);
         let thin_complete_streak_x = side_a_image.width() * 1 / 5;
         let thick_complete_streak_x = side_a_image.width() * 2 / 5;
@@ -877,7 +877,7 @@ mod test {
             }
             // Draw an incomplete streak on side B
             if y > 20 {
-                side_a_image.put_pixel(incomplete_streak_x, y, black_pixel);
+                side_b_image.put_pixel(incomplete_streak_x, y, black_pixel);
             }
             side_a_image.put_pixel(cropped_streak_x, y, black_pixel);
         }
@@ -899,6 +899,56 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_vertical_streak_through_left_timing_mark() {
+        let (mut side_a_image, side_b_image, options) =
+            load_hmpb_fixture("general-election/letter", 1);
+        let timing_mark_x = 60;
+        let black_pixel = Luma([0]);
+        for y in 0..side_a_image.height() {
+            side_a_image.put_pixel(timing_mark_x, y, black_pixel);
+        }
+        let Error::VerticalStreaksDetected {
+            label,
+            x_coordinates,
+        } = ballot_card(side_a_image, side_b_image, &options).unwrap_err()
+        else {
+            panic!("wrong error type");
+        };
+        assert_eq!(label, "side A");
+        assert_eq!(x_coordinates, vec![timing_mark_x as PixelPosition]);
+    }
+
+    #[test]
+    fn test_vertical_streak_through_right_timing_mark() {
+        let (mut side_a_image, side_b_image, options) =
+            load_hmpb_fixture("general-election/letter", 1);
+        let timing_mark_x = side_a_image.width() - 60;
+        let black_pixel = Luma([0]);
+        for y in 0..side_a_image.height() {
+            side_a_image.put_pixel(timing_mark_x, y, black_pixel);
+        }
+        let Error::VerticalStreaksDetected {
+            label,
+            x_coordinates,
+        } = ballot_card(side_a_image, side_b_image, &options).unwrap_err()
+        else {
+            panic!("wrong error type");
+        };
+        assert_eq!(label, "side A");
+        assert_eq!(x_coordinates, vec![timing_mark_x as PixelPosition]);
+    }
+
+    #[test]
+    fn test_ignore_edge_adjacent_vertical_streaks() {
+        let (side_a_image, side_b_image, options) = load_ballot_card_fixture(
+            "nh-test-ballot",
+            ("grayscale-front.png", "grayscale-back.png"),
+        );
+
+        ballot_card(side_a_image, side_b_image, &options).unwrap();
+    }
+
     #[test]
     fn test_rotated_ballot_scoring_write_in_areas_no_write_ins() {
         let (side_a_image, side_b_image, options) = load_hmpb_fixture("general-election/letter", 3);