diff --git a/apps/demo/src/pages/scrcpy.tsx b/apps/demo/src/pages/scrcpy.tsx index 9ea1d8294..637c861ad 100644 --- a/apps/demo/src/pages/scrcpy.tsx +++ b/apps/demo/src/pages/scrcpy.tsx @@ -40,9 +40,11 @@ import { AndroidKeyEventAction, AndroidMotionEventAction, AndroidScreenPowerMode, + clamp, CodecOptions, DEFAULT_SERVER_PATH, ScrcpyDeviceMessageType, + ScrcpyHoverHelper, ScrcpyLogLevel, ScrcpyOptions1_25, ScrcpyOptionsInit1_24, @@ -157,18 +159,6 @@ function fetchServer( return cachedValue.promise; } -function clamp(value: number, min: number, max: number): number { - if (value < min) { - return min; - } - - if (value > max) { - return max; - } - - return value; -} - export interface H264Decoder extends Disposable { readonly maxProfile: AndroidCodecProfile | undefined; readonly maxLevel: AndroidCodecLevel | undefined; @@ -258,6 +248,7 @@ const useClasses = makeStyles({ }, video: { transformOrigin: "center center", + touchAction: "none", }, }); @@ -369,6 +360,7 @@ class ScrcpyPageState { } client: AdbScrcpyClient | undefined = undefined; + hoverHelper: ScrcpyHoverHelper | undefined = undefined; async pushServer() { const serverBuffer = await fetchServer(); @@ -891,6 +883,7 @@ class ScrcpyPageState { handlePointerDown: false, handlePointerMove: false, handlePointerUp: false, + handlePointerLeave: false, handleWheel: false, handleContextMenu: false, handleKeyDown: false, @@ -1117,6 +1110,7 @@ class ScrcpyPageState { runInAction(() => { this.client = client; + this.hoverHelper = new ScrcpyHoverHelper(); this.running = true; }); } catch (e: any) { @@ -1301,37 +1295,46 @@ class ScrcpyPageState { const { pointerType } = e; let pointerId: bigint; - let { pressure } = e; if (pointerType === "mouse") { // ScrcpyPointerId.Mouse doesn't work with Chrome browser // https://github.com/Genymobile/scrcpy/issues/3635 pointerId = ScrcpyPointerId.Finger; - pressure = pressure === 0 ? 0 : 1; } else { pointerId = BigInt(e.pointerId); } const { x, y } = this.calculatePointerPosition(e.clientX, e.clientY); - this.client!.controlMessageSerializer!.injectTouch({ + + const messages = this.hoverHelper!.process({ action, pointerId, - screenWidth: this.client!.screenWidth!, - screenHeight: this.client!.screenHeight!, + screenWidth: this.client.screenWidth!, + screenHeight: this.client.screenHeight!, pointerX: x, pointerY: y, - pressure, + pressure: e.pressure, buttons: e.buttons, }); + for (const message of messages) { + this.client.controlMessageSerializer!.injectTouch(message); + } }; handlePointerDown = (e: React.PointerEvent) => { this.rendererContainer!.focus(); e.preventDefault(); + e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); this.injectTouch(AndroidMotionEventAction.Down, e); }; handlePointerMove = (e: React.PointerEvent) => { + if (!this.client) { + return; + } + + e.preventDefault(); + e.stopPropagation(); this.injectTouch( e.buttons === 0 ? AndroidMotionEventAction.HoverMove @@ -1341,6 +1344,16 @@ class ScrcpyPageState { }; handlePointerUp = (e: React.PointerEvent) => { + e.preventDefault(); + e.stopPropagation(); + this.injectTouch(AndroidMotionEventAction.Up, e); + }; + + handlePointerLeave = (e: React.PointerEvent) => { + e.preventDefault(); + e.stopPropagation(); + // Prevent hover state on device from "stucking" at the last position + this.injectTouch(AndroidMotionEventAction.HoverExit, e); this.injectTouch(AndroidMotionEventAction.Up, e); }; @@ -1358,8 +1371,8 @@ class ScrcpyPageState { screenHeight: this.client!.screenHeight!, pointerX: x, pointerY: y, - scrollX: e.deltaX / 100, - scrollY: e.deltaY / 100, + scrollX: -e.deltaX / 100, + scrollY: -e.deltaY / 100, buttons: 0, }); }; @@ -1591,6 +1604,7 @@ const Scrcpy: NextPage = () => { onPointerMove={state.handlePointerMove} onPointerUp={state.handlePointerUp} onPointerCancel={state.handlePointerUp} + onPointerLeave={state.handlePointerLeave} onKeyDown={state.handleKeyDown} onContextMenu={state.handleContextMenu} /> diff --git a/libraries/scrcpy/src/control/hover-helper.ts b/libraries/scrcpy/src/control/hover-helper.ts new file mode 100644 index 000000000..42af3d92a --- /dev/null +++ b/libraries/scrcpy/src/control/hover-helper.ts @@ -0,0 +1,51 @@ +import { + AndroidMotionEventAction, + type ScrcpyInjectTouchControlMessage, +} from "./inject-touch.js"; +import { ScrcpyControlMessageType } from "./type.js"; + +/** + * On Android, touching the screen with a finger will disable mouse cursor. + * However, Scrcpy doesn't do that, and can inject two pointers at the same time. + * This can cause finger events to be "ignored" because mouse is still the primary pointer. + * + * This helper class injects an extra `ACTION_UP` event, + * so Scrcpy server can remove the previously hovering pointer. + */ +export class ScrcpyHoverHelper { + // AFAIK, only mouse and pen can have hover state + // and you can't have two mouses or pens. + private lastHoverMessage: ScrcpyInjectTouchControlMessage | undefined; + + public process( + message: Omit + ): ScrcpyInjectTouchControlMessage[] { + const result: ScrcpyInjectTouchControlMessage[] = []; + + // A different pointer appeared, + // Cancel previously hovering pointer so Scrcpy server can free up the pointer ID. + if ( + this.lastHoverMessage && + this.lastHoverMessage.pointerId !== message.pointerId + ) { + // TODO: Inject MotionEvent.ACTION_HOVER_EXIT + // From testing, it seems no App cares about this event. + result.push({ + ...this.lastHoverMessage, + action: AndroidMotionEventAction.Up, + }); + this.lastHoverMessage = undefined; + } + + if (message.action === AndroidMotionEventAction.HoverMove) { + // TODO: Inject MotionEvent.ACTION_HOVER_ENTER + this.lastHoverMessage = message as ScrcpyInjectTouchControlMessage; + } + + (message as ScrcpyInjectTouchControlMessage).type = + ScrcpyControlMessageType.InjectTouch; + result.push(message as ScrcpyInjectTouchControlMessage); + + return result; + } +} diff --git a/libraries/scrcpy/src/control/index.ts b/libraries/scrcpy/src/control/index.ts index 03f5d7c97..59c25416c 100644 --- a/libraries/scrcpy/src/control/index.ts +++ b/libraries/scrcpy/src/control/index.ts @@ -1,4 +1,5 @@ export * from "./back-or-screen-on.js"; +export * from "./hover-helper.js"; export * from "./inject-keycode.js"; export * from "./inject-scroll.js"; export * from "./inject-text.js"; diff --git a/libraries/scrcpy/src/control/inject-touch.ts b/libraries/scrcpy/src/control/inject-touch.ts index 026832146..c406b481d 100644 --- a/libraries/scrcpy/src/control/inject-touch.ts +++ b/libraries/scrcpy/src/control/inject-touch.ts @@ -31,6 +31,18 @@ export namespace ScrcpyPointerId { export const VirtualFinger = BigInt(-4); } +export function clamp(value: number, min: number, max: number): number { + if (value < min) { + return min; + } + + if (value > max) { + return max; + } + + return value; +} + const Uint16Max = (1 << 16) - 1; const ScrcpyFloatToUint16NumberType: NumberFieldType = { @@ -41,7 +53,7 @@ const ScrcpyFloatToUint16NumberType: NumberFieldType = { return value / Uint16Max; }, serialize(dataView, offset, value, littleEndian) { - value = value * Uint16Max; + value = clamp(value, 0, 1) * Uint16Max; NumberFieldType.Uint16.serialize(dataView, offset, value, littleEndian); }, }; diff --git a/libraries/scrcpy/src/options/1_16/scroll.ts b/libraries/scrcpy/src/options/1_16/scroll.ts index e709d0b2c..e37190d7f 100644 --- a/libraries/scrcpy/src/options/1_16/scroll.ts +++ b/libraries/scrcpy/src/options/1_16/scroll.ts @@ -40,18 +40,18 @@ export class ScrcpyScrollController1_16 implements ScrcpyScrollController { let scrollY = 0; if (this.accumulatedX >= 1) { scrollX = 1; - this.accumulatedX -= 1; + this.accumulatedX = 0; } else if (this.accumulatedX <= -1) { scrollX = -1; - this.accumulatedX += 1; + this.accumulatedX = 0; } if (this.accumulatedY >= 1) { scrollY = 1; - this.accumulatedY -= 1; + this.accumulatedY = 0; } else if (this.accumulatedY <= -1) { scrollY = -1; - this.accumulatedY += 1; + this.accumulatedY = 0; } if (scrollX === 0 && scrollY === 0) { diff --git a/libraries/scrcpy/src/options/1_25/scroll.ts b/libraries/scrcpy/src/options/1_25/scroll.ts index de1992071..35290c330 100644 --- a/libraries/scrcpy/src/options/1_25/scroll.ts +++ b/libraries/scrcpy/src/options/1_25/scroll.ts @@ -4,6 +4,7 @@ import Struct, { } from "@yume-chan/struct"; import { + clamp, ScrcpyControlMessageType, type ScrcpyInjectScrollControlMessage, } from "../../control/index.js"; @@ -19,7 +20,7 @@ const ScrcpyFloatToInt16NumberType: NumberFieldType = { return value / Int16Max; }, serialize(dataView, offset, value, littleEndian) { - value = value * Int16Max; + value = clamp(value, -1, 1) * Int16Max; NumberFieldType.Int16.serialize(dataView, offset, value, littleEndian); }, };