diff --git a/app/src/input_events.h b/app/src/input_events.h
index ed77bcb4d8..bbf4372fa2 100644
--- a/app/src/input_events.h
+++ b/app/src/input_events.h
@@ -437,25 +437,11 @@ sc_mouse_button_from_sdl(uint8_t button) {
 }
 
 static inline uint8_t
-sc_mouse_buttons_state_from_sdl(uint32_t buttons_state,
-                                const struct sc_mouse_bindings *mb) {
+sc_mouse_buttons_state_from_sdl(uint32_t buttons_state) {
     assert(buttons_state < 0x100); // fits in uint8_t
 
-    uint8_t mask = SC_MOUSE_BUTTON_LEFT;
-    if (!mb || mb->right_click == SC_MOUSE_BINDING_CLICK) {
-        mask |= SC_MOUSE_BUTTON_RIGHT;
-    }
-    if (!mb || mb->middle_click == SC_MOUSE_BINDING_CLICK) {
-        mask |= SC_MOUSE_BUTTON_MIDDLE;
-    }
-    if (!mb || mb->click4 == SC_MOUSE_BINDING_CLICK) {
-        mask |= SC_MOUSE_BUTTON_X1;
-    }
-    if (!mb || mb->click5 == SC_MOUSE_BINDING_CLICK) {
-        mask |= SC_MOUSE_BUTTON_X2;
-    }
-
-    return buttons_state & mask;
+    // SC_MOUSE_BUTTON_* constants are initialized from SDL_BUTTON(index)
+    return buttons_state;
 }
 
 #endif
diff --git a/app/src/input_manager.c b/app/src/input_manager.c
index 6fbd801c95..2cc34afac9 100644
--- a/app/src/input_manager.c
+++ b/app/src/input_manager.c
@@ -77,6 +77,8 @@ sc_input_manager_init(struct sc_input_manager *im,
     im->vfinger_invert_x = false;
     im->vfinger_invert_y = false;
 
+    im->mouse_buttons_state = 0;
+
     im->last_keycode = SDLK_UNKNOWN;
     im->last_mod = 0;
     im->key_repeat = 0;
@@ -654,8 +656,7 @@ sc_input_manager_process_mouse_motion(struct sc_input_manager *im,
                                        : SC_POINTER_ID_MOUSE,
         .xrel = event->xrel,
         .yrel = event->yrel,
-        .buttons_state =
-            sc_mouse_buttons_state_from_sdl(event->state, &im->mouse_bindings),
+        .buttons_state = im->mouse_buttons_state,
     };
 
     assert(im->mp->ops->process_mouse_motion);
@@ -736,6 +737,13 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im,
     bool control = im->controller;
     bool paused = im->screen->paused;
     bool down = event->type == SDL_MOUSEBUTTONDOWN;
+
+    enum sc_mouse_button button = sc_mouse_button_from_sdl(event->button);
+    if (!down) {
+        // Mark the button as released
+        im->mouse_buttons_state &= ~button;
+    }
+
     if (control && !paused) {
         enum sc_action action = down ? SC_ACTION_DOWN : SC_ACTION_UP;
 
@@ -799,7 +807,10 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im,
         return;
     }
 
-    uint32_t sdl_buttons_state = SDL_GetMouseState(NULL, NULL);
+    if (down) {
+        // Mark the button as pressed
+        im->mouse_buttons_state |= button;
+    }
 
     SDL_Keymod keymod = SDL_GetModState();
     bool ctrl_pressed = keymod & KMOD_CTRL;
@@ -815,8 +826,7 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im,
         .button = sc_mouse_button_from_sdl(event->button),
         .pointer_id = use_finger ? SC_POINTER_ID_GENERIC_FINGER
                                  : SC_POINTER_ID_MOUSE,
-        .buttons_state = sc_mouse_buttons_state_from_sdl(sdl_buttons_state,
-                                                         &im->mouse_bindings),
+        .buttons_state = im->mouse_buttons_state,
     };
 
     assert(im->mp->ops->process_mouse_click);
@@ -875,6 +885,7 @@ sc_input_manager_process_mouse_wheel(struct sc_input_manager *im,
     int mouse_x;
     int mouse_y;
     uint32_t buttons = SDL_GetMouseState(&mouse_x, &mouse_y);
+    (void) buttons; // Actual buttons are tracked manually to ignore shortcuts
 
     struct sc_mouse_scroll_event evt = {
         .position = sc_input_manager_get_position(im, mouse_x, mouse_y),
@@ -885,8 +896,7 @@ sc_input_manager_process_mouse_wheel(struct sc_input_manager *im,
         .hscroll = CLAMP(event->x, -1, 1),
         .vscroll = CLAMP(event->y, -1, 1),
 #endif
-        .buttons_state = sc_mouse_buttons_state_from_sdl(buttons,
-                                                         &im->mouse_bindings),
+        .buttons_state = im->mouse_buttons_state,
     };
 
     im->mp->ops->process_mouse_scroll(im->mp, &evt);
diff --git a/app/src/input_manager.h b/app/src/input_manager.h
index d5a5a64d3f..88558549b3 100644
--- a/app/src/input_manager.h
+++ b/app/src/input_manager.h
@@ -32,6 +32,8 @@ struct sc_input_manager {
     bool vfinger_invert_x;
     bool vfinger_invert_y;
 
+    uint8_t mouse_buttons_state; // OR of enum sc_mouse_button values
+
     // Tracks the number of identical consecutive shortcut key down events.
     // Not to be confused with event->repeat, which counts the number of
     // system-generated repeated key presses.
diff --git a/app/src/usb/screen_otg.c b/app/src/usb/screen_otg.c
index 33500e0c23..5c4f97f0f7 100644
--- a/app/src/usb/screen_otg.c
+++ b/app/src/usb/screen_otg.c
@@ -169,7 +169,7 @@ sc_screen_otg_process_mouse_motion(struct sc_screen_otg *screen,
         // .position not used for HID events
         .xrel = event->xrel,
         .yrel = event->yrel,
-        .buttons_state = sc_mouse_buttons_state_from_sdl(event->state, NULL),
+        .buttons_state = sc_mouse_buttons_state_from_sdl(event->state),
     };
 
     assert(mp->ops->process_mouse_motion);
@@ -188,8 +188,7 @@ sc_screen_otg_process_mouse_button(struct sc_screen_otg *screen,
         // .position not used for HID events
         .action = sc_action_from_sdl_mousebutton_type(event->type),
         .button = sc_mouse_button_from_sdl(event->button),
-        .buttons_state =
-            sc_mouse_buttons_state_from_sdl(sdl_buttons_state, NULL),
+        .buttons_state = sc_mouse_buttons_state_from_sdl(sdl_buttons_state),
     };
 
     assert(mp->ops->process_mouse_click);
@@ -208,8 +207,7 @@ sc_screen_otg_process_mouse_wheel(struct sc_screen_otg *screen,
         // .position not used for HID events
         .hscroll = event->x,
         .vscroll = event->y,
-        .buttons_state =
-            sc_mouse_buttons_state_from_sdl(sdl_buttons_state, NULL),
+        .buttons_state = sc_mouse_buttons_state_from_sdl(sdl_buttons_state),
     };
 
     assert(mp->ops->process_mouse_scroll);