Merge pull request #1123 from anatawa12/fix-broken-with-negative-uv

fix: RemoveMeshByMask is broken with negative uv
anatawa12 · Jul 25, 2024 · 13e3fbe · 13e3fbe
2 parents deaedd2 + 3f4b372
commit 13e3fbe
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 4 deletions.
diff --git a/CHANGELOG-PRERELEASE.md b/CHANGELOG-PRERELEASE.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog].
 ### Removed
 
 ### Fixed
+- Index out of bounds error with remove mesh by mask with negative UV `#1123`
 
 ### Security
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog].
 ### Removed
 
 ### Fixed
+- Index out of bounds error with remove mesh by mask with negative UV `#1123`
 
 ### Security
 

diff --git a/Editor/EditModePreview/RemoveMeshPreviewController.cs b/Editor/EditModePreview/RemoveMeshPreviewController.cs
@@ -453,8 +453,8 @@ private bool TestBlendShape(int movementBase, int index) =>
 
             private int GetValue(Vector2 uv)
             {
-                var x = Mathf.FloorToInt(uv.x % 1 * MaskWidth);
-                var y = Mathf.FloorToInt(uv.y % 1 * MaskHeight);
+                var x = Mathf.FloorToInt(Utils.Modulo(uv.x, 1) * MaskWidth);
+                var y = Mathf.FloorToInt(Utils.Modulo(uv.y, 1) * MaskHeight);
                 var color = Mask[x + y * MaskWidth];
                 return Mathf.Max(Mathf.Max(color.r, color.g), color.b);
             }

diff --git a/Editor/Processors/SkinnedMeshes/RemoveMeshByMaskProcessor.cs b/Editor/Processors/SkinnedMeshes/RemoveMeshByMaskProcessor.cs
@@ -75,8 +75,9 @@ void AutoFix()
 
                 int GetValue(float u, float v)
                 {
-                    var x = Mathf.FloorToInt(u % 1 * textureWidth);
-                    var y = Mathf.FloorToInt(v % 1 * textureHeight);
+                    var x = Mathf.FloorToInt(Utils.Modulo(u, 1) * textureWidth);
+                    var y = Mathf.FloorToInt(Utils.Modulo(v, 1) * textureHeight);
+                    if (y * textureWidth + x < 0 || y * textureWidth + x >= pixels.Length) throw new IndexOutOfRangeException($"x: {x}, y: {y}, u: {u}, v: {v}, w: {textureWidth}, h: {textureHeight}, l: {pixels.Length}");
                     var pixel = pixels[y * textureWidth + x];
                     return Mathf.Max(Mathf.Max(pixel.r, pixel.g), pixel.b);
                 }

diff --git a/Internal/Utils/Utils.Float.cs b/Internal/Utils/Utils.Float.cs
@@ -1,5 +1,6 @@
 
 using System;
+using UnityEngine;
 
 namespace Anatawa12.AvatarOptimizer
 {
@@ -91,5 +92,6 @@ public static float PreviousFloat(float x)
 #endif
 
         public static bool IsFinite(float x) => !float.IsNaN(x) && !float.IsInfinity(x);
+        public static float Modulo(float x, float y) => x - y * Mathf.Floor(x / y);
     }
 }