core: Fix mouse coords reported to ActionScript (#14243)

The order in which Flash Player concatenates matrices causes reported mouse coords to often be rounded.
ruffle-rs · Dec 15, 2023 · f427cd5 · f427cd5
1 parent d21be20
commit f427cd5
Show file tree

Hide file tree

Showing 18 changed files with 1,173 additions and 12 deletions.
diff --git a/core/src/avm1/object/stage_object.rs b/core/src/avm1/object/stage_object.rs
@@ -798,12 +798,12 @@ fn set_quality<'gc>(
 }
 
 fn x_mouse<'gc>(activation: &mut Activation<'_, 'gc>, this: DisplayObject<'gc>) -> Value<'gc> {
-    let local = this.mouse_to_local(*activation.context.mouse_position);
+    let local = this.local_mouse_position(&activation.context);
     local.x.to_pixels().into()
 }
 
 fn y_mouse<'gc>(activation: &mut Activation<'_, 'gc>, this: DisplayObject<'gc>) -> Value<'gc> {
-    let local = this.mouse_to_local(*activation.context.mouse_position);
+    let local = this.local_mouse_position(&activation.context);
     local.y.to_pixels().into()
 }
 

diff --git a/core/src/avm2/globals/flash/display/display_object.rs b/core/src/avm2/globals/flash/display/display_object.rs
@@ -631,7 +631,7 @@ pub fn get_mouse_x<'gc>(
     _args: &[Value<'gc>],
 ) -> Result<Value<'gc>, Error<'gc>> {
     if let Some(dobj) = this.as_display_object() {
-        let local_mouse = dobj.mouse_to_local(*activation.context.mouse_position);
+        let local_mouse = dobj.local_mouse_position(&activation.context);
         return Ok(local_mouse.x.to_pixels().into());
     }
 
@@ -645,7 +645,7 @@ pub fn get_mouse_y<'gc>(
     _args: &[Value<'gc>],
 ) -> Result<Value<'gc>, Error<'gc>> {
     if let Some(dobj) = this.as_display_object() {
-        let local_mouse = dobj.mouse_to_local(*activation.context.mouse_position);
+        let local_mouse = dobj.local_mouse_position(&activation.context);
         return Ok(local_mouse.y.to_pixels().into());
     }
 

diff --git a/core/src/avm2/object/event_object.rs b/core/src/avm2/object/event_object.rs
@@ -104,7 +104,7 @@ impl<'gc> EventObject<'gc> {
     where
         S: Into<AvmString<'gc>>,
     {
-        let local = target.mouse_to_local(*activation.context.mouse_position);
+        let local = target.local_mouse_position(&activation.context);
 
         let event_type: AvmString<'gc> = event_type.into();
 

diff --git a/core/src/display_object.rs b/core/src/display_object.rs
@@ -1248,15 +1248,30 @@ pub trait TDisplayObject<'gc>:
         self.global_to_local_matrix().map(|matrix| matrix * global)
     }
 
-    /// Converts a mouse position on the stage to a local position on this display object.
+    /// Converts the mouse position on the stage to a local position on this display object.
     /// If the object has zero scale, then the stage `TWIPS_TO_PIXELS` matrix will be used.
     /// This matches Flash's behavior for `mouseX`/`mouseY` on an object with zero scale.
-    fn mouse_to_local(&self, global: Point<Twips>) -> Point<Twips> {
-        // MIKE: I suspect the `TWIPS_TO_PIXELS` scale should always be involved in the
-        // calculation somehow, not just in the non-invertible case.
-        self.global_to_local_matrix()
-            .unwrap_or(Matrix::TWIPS_TO_PIXELS)
-            * global
+    fn local_mouse_position(&self, context: &UpdateContext<'_, 'gc>) -> Point<Twips> {
+        let stage = context.stage;
+        let pixel_ratio = stage.view_matrix().a;
+        let virtual_to_device = Matrix::scale(pixel_ratio, pixel_ratio);
+
+        // Get mouse pos in global device pixels
+        let global_twips = *context.mouse_position;
+        let global_device_twips = virtual_to_device * global_twips;
+        let global_device_pixels = Matrix::TWIPS_TO_PIXELS * global_device_twips;
+
+        // Make transformation matrix
+        let local_twips_to_global_twips = self.local_to_global_matrix();
+        let twips_to_device_pixels = virtual_to_device * Matrix::TWIPS_TO_PIXELS;
+        let local_twips_to_global_device_pixels =
+            twips_to_device_pixels * local_twips_to_global_twips;
+        let global_device_pixels_to_local_twips = local_twips_to_global_device_pixels
+            .inverse()
+            .unwrap_or(Matrix::IDENTITY);
+
+        // Get local mouse position in twips
+        global_device_pixels_to_local_twips * global_device_pixels
     }
 
     /// The `x` position in pixels of this display object in local space.

diff --git a/tests/tests/swfs/avm1/mouse_pos/input.json b/tests/tests/swfs/avm1/mouse_pos/input.json
@@ -0,0 +1,76 @@
+[
+  {"type": "MouseMove", "pos": [308, 340]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [309, 340]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [309, 339]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [309, 330]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [309, 307]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [306, 265]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [297, 236]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [289, 195]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [288, 167]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [307, 110]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [328, 82]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [352, 65]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [362, 64]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [368, 59]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [381, 52]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [404, 49]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [417, 58]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [430, 82]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [435, 109]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [435, 139]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [429, 163]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [419, 192]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [410, 208]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [393, 232]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [378, 247]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [333, 269]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [305, 281]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [262, 301]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [236, 309]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [201, 312]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [178, 300]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [147, 271]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [127, 251]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [122, 241]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [122, 234]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [122, 212]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [111, 165]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [101, 146]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [84, 119]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [71, 108]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [52, 98]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [39, 90]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [29, 75]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [27, 68]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [26, 50]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [26, 37]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [29, 23]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [30, 16]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [33, 12]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [35, 11]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [38, 8]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [48, 6]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [92, 6]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [143, 14]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [237, 52]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [293, 84]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [352, 110]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [379, 113]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [401, 113]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [407, 120]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [406, 150]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [388, 171]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [348, 193]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [326, 195]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [315, 195]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [315, 194]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [340, 153]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [431, 56]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [468, 30]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [491, 19]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [499, 16]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [508, 11]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [511, 2]}, {"type": "Wait"},
+  {"type": "MouseMove", "pos": [512, 0]}, {"type": "Wait"}
+]
diff --git a/tests/tests/swfs/avm1/mouse_pos/input.json.py b/tests/tests/swfs/avm1/mouse_pos/input.json.py
@@ -0,0 +1,15 @@
+# You can use this to recreate input.json from Flash Player output if necessary.
+
+import re
+
+first = True
+with open('output.txt', 'rt') as f:
+    print('[')
+    for line in f:
+        if m := re.match('^_root (\d+) (\d+)', line):
+            x, y = m.groups()
+            if not first:
+                print(',')
+            print(f'  {{"type": "MouseMove", "pos": [{x}, {y}]}}, {{"type": "Wait"}}', end='')
+            first = False
+    print('\n]')