Add HID keyboard simulation via USB HID over AoAv2

BUILD.md

@@ -88,11 +88,12 @@ Install the required packages from your package manager.
 
 ```bash
 # runtime dependencies
-sudo apt install ffmpeg libsdl2-2.0-0 adb
+sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
 
 # client build dependencies
 sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
-                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev\
+                 libusb-dev
 
 # server build dependencies
 sudo apt install openjdk-11-jdk
@@ -114,7 +115,7 @@ pip3 install meson
 sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
 
 # client build dependencies
-sudo dnf install SDL2-devel ffms2-devel meson gcc make
+sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
 
 # server build dependencies
 sudo dnf install java-devel
@@ -159,7 +160,8 @@ install the required packages:
 ```bash
 # runtime dependencies
 pacman -S mingw-w64-x86_64-SDL2 \
-          mingw-w64-x86_64-ffmpeg
+          mingw-w64-x86_64-ffmpeg \
+          mingw-w64-x86_64-libusb
 
 # client build dependencies
 pacman -S mingw-w64-x86_64-make \
@@ -173,7 +175,8 @@ For a 32 bits version, replace `x86_64` by `i686`:
 ```bash
 # runtime dependencies
 pacman -S mingw-w64-i686-SDL2 \
-          mingw-w64-i686-ffmpeg
+          mingw-w64-i686-ffmpeg \
+          mingw-w64-i686-libusb
 
 # client build dependencies
 pacman -S mingw-w64-i686-make \
@@ -197,7 +200,7 @@ Install the packages with [Homebrew]:
 
 ```bash
 # runtime dependencies
-brew install sdl2 ffmpeg
+brew install sdl2 ffmpeg libusb
 
 # client build dependencies
 brew install pkg-config meson

README.md

@@ -207,6 +207,29 @@ scrcpy --crop 1224:1440:0:0   # 1224x1440 at offset (0,0)
 
 If `--max-size` is also specified, resizing is applied after cropping.
 
+#### USB HID over AOAv2
+
+Scrcpy can simulate a USB physical keyboard on Android to provide better input
+experience, you need to connect your device via USB, not wireless.
+
+However, due to some limitation of libusb and WinUSB driver, you cannot use HID
+over AOAv2 on Windows.
+
+Currently a USB serial number is needed to use HID over AOAv2.
+
+```bash
+scrcpy --serial XXXXXXXXXXXXXXXX # don't use HID
+scrcpy --serial XXXXXXXXXXXXXXXX --input-mode inject # don't use HID
+scrcpy --serial XXXXXXXXXXXXXXXX --input-mode hid # try HID and exit if failed
+```
+
+Serial number can be found by `adb get-serialno`.
+
+If you are a non-QWERTY keyboard user and using HID mode, please remember to set
+correct physical keyboard layout manually in Android settings, because scrcpy
+just forwards scancodes to Android device and Android system is responsible for
+converting scancodes to correct keycode on Android device (your system does this
+on your PC).
 
 #### Lock video orientation
 

app/meson.build

@@ -1,6 +1,7 @@
 src = [
     'src/main.c',
     'src/adb.c',
+    'src/aoa_hid.c',
     'src/cli.c',
     'src/clock.c',
     'src/compat.c',
@@ -12,6 +13,7 @@ src = [
     'src/file_handler.c',
     'src/fps_counter.c',
     'src/frame_buffer.c',
+    'src/hid_keyboard.c',
     'src/input_manager.c',
     'src/opengl.c',
     'src/receiver.c',
@@ -54,6 +56,7 @@ if not get_option('crossbuild_windows')
         dependency('libavformat'),
         dependency('libavcodec'),
         dependency('libavutil'),
+        dependency('libusb-1.0'),
         dependency('sdl2'),
     ]
 
@@ -90,8 +93,20 @@ else
         include_directories: include_directories(ffmpeg_include_dir)
     )
 
+    prebuilt_libusb = meson.get_cross_property('prebuilt_libusb')
+    libusb_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_libusb + '/dll'
+    libusb_include_dir = '../prebuilt-deps/' + prebuilt_libusb + '/include'
+
+    libusb = declare_dependency(
+        dependencies: [
+            cc.find_library('libusb-1.0', dirs: libusb_bin_dir),
+        ],
+        include_directories: include_directories(libusb_include_dir)
+    )
+
     dependencies = [
         ffmpeg,
+        libusb,
         sdl2,
         cc.find_library('mingw32')
     ]

app/scrcpy.1

@@ -82,6 +82,16 @@ Start in fullscreen.
 .B \-h, \-\-help
 Print this help.
 
+.TP
+.B \-i, \-\-input\-mode mode
+Select input mode for keyboard events.
+
+Possible values are "hid" and "inject".
+
+"hid" uses Android's USB HID over AOAv2 feature to simulate physical keyboard's events, which provides better experience for IME users if supported by you device.
+
+"inject" is the legacy scrcpy way by injecting keycode events on Android, works on most devices and is the default.
+
 .TP
 .B \-\-legacy\-paste
 Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).