From 91418c79ab3b7056c21ef6c64f903704c8f00a52 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 22 Jan 2022 11:09:41 +0100 Subject: [PATCH] Add OTG mode Add an option --otg to run scrcpy with only physical keyboard and mouse simulation (HID over AOA), without mirroring and without requiring adb. To avoid adding complexity into the scrcpy initialization and screen implementation, OTG mode is implemented totally separately, with a separate window. PR #2974 --- app/meson.build | 2 + app/scrcpy.1 | 12 ++ app/src/cli.c | 61 ++++++++++ app/src/events.h | 1 + app/src/main.c | 10 +- app/src/options.c | 3 + app/src/options.h | 3 + app/src/usb/scrcpy_otg.c | 213 ++++++++++++++++++++++++++++++++ app/src/usb/scrcpy_otg.h | 12 ++ app/src/usb/screen_otg.c | 254 +++++++++++++++++++++++++++++++++++++++ app/src/usb/screen_otg.h | 46 +++++++ 11 files changed, 615 insertions(+), 2 deletions(-) create mode 100644 app/src/usb/scrcpy_otg.c create mode 100644 app/src/usb/scrcpy_otg.h create mode 100644 app/src/usb/screen_otg.c create mode 100644 app/src/usb/screen_otg.h diff --git a/app/meson.build b/app/meson.build index 5497281a4a..80a0d9d280 100644 --- a/app/meson.build +++ b/app/meson.build @@ -78,6 +78,8 @@ if usb_support 'src/usb/aoa_hid.c', 'src/usb/hid_keyboard.c', 'src/usb/hid_mouse.c', + 'src/usb/scrcpy_otg.c', + 'src/usb/screen_otg.c', 'src/usb/usb.c', ] endif diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 37431a91cb..e8e74f987f 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -162,6 +162,18 @@ Do not forward repeated key events when a key is held down. .B \-\-no\-mipmaps If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then mipmaps are automatically generated to improve downscaling quality. This option disables the generation of mipmaps. +.TP +.B \-\-otg +Run in OTG mode: simulate physical keyboard and mouse, as if the computer keyboard and mouse were plugged directly to the device via an OTG cable. + +In this mode, adb (USB debugging) is not necessary, and mirroring is disabled. + +LAlt, LSuper or RSuper toggle the mouse capture mode, to give control of the mouse back to the computer. + +It may only work over USB, and is currently only supported on Linux. + +See \fB\-\-hid\-keyboard\fR and \fB\-\-hid\-mouse\fR. + .TP .BI "\-p, \-\-port " port[:port] Set the TCP port (range) used by the client to listen. diff --git a/app/src/cli.c b/app/src/cli.c index 34c3103abb..28462efd97 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -53,6 +53,7 @@ #define OPT_TCPIP 1033 #define OPT_RAW_KEY_EVENTS 1034 #define OPT_NO_DOWNSIZE_ON_ERROR 1035 +#define OPT_OTG 1036 struct sc_option { char shortopt; @@ -276,6 +277,20 @@ static const struct sc_option options[] = { "mipmaps are automatically generated to improve downscaling " "quality. This option disables the generation of mipmaps.", }, + { + .longopt_id = OPT_OTG, + .longopt = "otg", + .text = "Run in OTG mode: simulate physical keyboard and mouse, " + "as if the computer keyboard and mouse were plugged directly " + "to the device via an OTG cable.\n" + "In this mode, adb (USB debugging) is not necessary, and " + "mirroring is disabled.\n" + "LAlt, LSuper or RSuper toggle the mouse capture mode, to give " + "control of the mouse back to the computer.\n" + "It may only work over USB, and is currently only supported " + "on Linux.\n" + "See --hid-keyboard and --hid-mouse.", + }, { .shortopt = 'p', .longopt = "port", @@ -1500,6 +1515,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_NO_DOWNSIZE_ON_ERROR: opts->downsize_on_error = false; break; + case OPT_OTG: +#ifdef HAVE_USB + opts->otg = true; + break; +#else + LOGE("OTG mode (--otg) is not supported on this platform. It " + "is only available on Linux."); + return false; +#endif case OPT_V4L2_SINK: #ifdef HAVE_V4L2 opts->v4l2_device = optarg; @@ -1610,6 +1634,43 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } +#ifdef HAVE_USB + if (opts->otg) { + // OTG mode is compatible with only very few options. + // Only report obvious errors. + if (opts->record_filename) { + LOGE("OTG mode: could not record"); + return false; + } + if (opts->turn_screen_off) { + LOGE("OTG mode: could not turn screen off"); + return false; + } + if (opts->stay_awake) { + LOGE("OTG mode: could not stay awake"); + return false; + } + if (opts->show_touches) { + LOGE("OTG mode: could not request to show touches"); + return false; + } + if (opts->power_off_on_close) { + LOGE("OTG mode: could not request power off on close"); + return false; + } + if (opts->display_id) { + LOGE("OTG mode: could not select display"); + return false; + } +#ifdef HAVE_V4L2 + if (opts->v4l2_device) { + LOGE("OTG mode: could not sink to V4L2 device"); + return false; + } +#endif + } +#endif + return true; } diff --git a/app/src/events.h b/app/src/events.h index abe1a72c86..3c14f96e51 100644 --- a/app/src/events.h +++ b/app/src/events.h @@ -2,3 +2,4 @@ #define EVENT_STREAM_STOPPED (SDL_USEREVENT + 1) #define EVENT_SERVER_CONNECTION_FAILED (SDL_USEREVENT + 2) #define EVENT_SERVER_CONNECTED (SDL_USEREVENT + 3) +#define EVENT_USB_DEVICE_DISCONNECTED (SDL_USEREVENT + 4) diff --git a/app/src/main.c b/app/src/main.c index cbcef4a798..8a8c029cc5 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -13,6 +13,7 @@ #include "cli.h" #include "options.h" #include "scrcpy.h" +#include "usb/scrcpy_otg.h" #include "util/log.h" static void @@ -88,9 +89,14 @@ main(int argc, char *argv[]) { return 1; } - int res = scrcpy(&args.opts) ? 0 : 1; +#ifdef HAVE_USB + bool ok = args.opts.otg ? scrcpy_otg(&args.opts) + : scrcpy(&args.opts); +#else + bool ok = scrcpy(&args.opts); +#endif avformat_network_deinit(); // ignore failure - return res; + return ok ? 0 : 1; } diff --git a/app/src/options.c b/app/src/options.c index b8560406fc..c94f798d20 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -37,6 +37,9 @@ const struct scrcpy_options scrcpy_options_default = { .display_id = 0, .display_buffer = 0, .v4l2_buffer = 0, +#ifdef HAVE_USB + .otg = false, +#endif .show_touches = false, .fullscreen = false, .always_on_top = false, diff --git a/app/src/options.h b/app/src/options.h index 99f03d4053..1591a065e1 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -112,6 +112,9 @@ struct scrcpy_options { uint32_t display_id; sc_tick display_buffer; sc_tick v4l2_buffer; +#ifdef HAVE_USB + bool otg; +#endif bool show_touches; bool fullscreen; bool always_on_top; diff --git a/app/src/usb/scrcpy_otg.c b/app/src/usb/scrcpy_otg.c new file mode 100644 index 0000000000..0a28eda4a1 --- /dev/null +++ b/app/src/usb/scrcpy_otg.c @@ -0,0 +1,213 @@ +#include "scrcpy_otg.h" + +#include + +#include "events.h" +#include "screen_otg.h" +#include "util/log.h" + +struct scrcpy_otg { + struct sc_usb usb; + struct sc_aoa aoa; + struct sc_hid_keyboard keyboard; + struct sc_hid_mouse mouse; + + struct sc_screen_otg screen_otg; +}; + +static void +sc_usb_on_disconnected(struct sc_usb *usb, void *userdata) { + (void) usb; + (void) userdata; + + SDL_Event event; + event.type = EVENT_USB_DEVICE_DISCONNECTED; + int ret = SDL_PushEvent(&event); + if (ret < 0) { + LOGE("Could not post USB disconnection event: %s", SDL_GetError()); + } +} + +static bool +event_loop(struct scrcpy_otg *s) { + SDL_Event event; + while (SDL_WaitEvent(&event)) { + switch (event.type) { + case EVENT_USB_DEVICE_DISCONNECTED: + LOGW("Device disconnected"); + return false; + case SDL_QUIT: + LOGD("User requested to quit"); + return true; + default: + sc_screen_otg_handle_event(&s->screen_otg, &event); + break; + } + } + return false; +} + +bool +scrcpy_otg(struct scrcpy_options *options) { + static struct scrcpy_otg scrcpy_otg; + struct scrcpy_otg *s = &scrcpy_otg; + + const char *serial = options->serial; + + // Minimal SDL initialization + if (SDL_Init(SDL_INIT_EVENTS)) { + LOGC("Could not initialize SDL: %s", SDL_GetError()); + return false; + } + + atexit(SDL_Quit); + + bool ret = false; + + struct sc_hid_keyboard *keyboard = NULL; + struct sc_hid_mouse *mouse = NULL; + bool usb_device_initialized = false; + bool usb_connected = false; + bool aoa_started = false; + bool aoa_initialized = false; + + static const struct sc_usb_callbacks cbs = { + .on_disconnected = sc_usb_on_disconnected, + }; + bool ok = sc_usb_init(&s->usb); + if (!ok) { + return false; + } + + struct sc_usb_device usb_devices[16]; + ssize_t count = sc_usb_find_devices(&s->usb, serial, usb_devices, + ARRAY_LEN(usb_devices)); + if (count < 0) { + LOGE("Could not list USB devices"); + goto end; + } + + if (count == 0) { + if (serial) { + LOGE("Could not find USB device %s", serial); + } else { + LOGE("Could not find any USB device"); + } + goto end; + } + + if (count > 1) { + if (serial) { + LOGE("Multiple (%d) USB devices with serial %s:", (int) count, + serial); + } else { + LOGE("Multiple (%d) USB devices:", (int) count); + } + for (size_t i = 0; i < (size_t) count; ++i) { + struct sc_usb_device *d = &usb_devices[i]; + LOGE(" %-18s (%04" PRIx16 ":%04" PRIx16 ") %s %s", + d->serial, d->vid, d->pid, d->manufacturer, d->product); + } + if (!serial) { + LOGE("Specify the device via -s or --serial"); + } + sc_usb_device_destroy_all(usb_devices, count); + goto end; + } + usb_device_initialized = true; + + struct sc_usb_device *usb_device = &usb_devices[0]; + + LOGI("USB device: %s (%04" PRIx16 ":%04" PRIx16 ") %s %s", + usb_device->serial, usb_device->vid, usb_device->pid, + usb_device->manufacturer, usb_device->product); + + ok = sc_usb_connect(&s->usb, usb_device->device, &cbs, NULL); + if (!ok) { + goto end; + } + usb_connected = true; + + ok = sc_aoa_init(&s->aoa, &s->usb, NULL); + if (!ok) { + goto end; + } + aoa_initialized = true; + + ok = sc_hid_keyboard_init(&s->keyboard, &s->aoa); + if (!ok) { + goto end; + } + keyboard = &s->keyboard; + + ok = sc_hid_mouse_init(&s->mouse, &s->aoa); + if (!ok) { + goto end; + } + mouse = &s->mouse; + + ok = sc_aoa_start(&s->aoa); + if (!ok) { + goto end; + } + aoa_started = true; + + const char *window_title = options->window_title; + if (!window_title) { + window_title = usb_device->product ? usb_device->product : "scrcpy"; + } + + struct sc_screen_otg_params params = { + .keyboard = keyboard, + .mouse = mouse, + .window_title = window_title, + .always_on_top = options->always_on_top, + .window_x = options->window_x, + .window_y = options->window_y, + .window_borderless = options->window_borderless, + }; + + ok = sc_screen_otg_init(&s->screen_otg, ¶ms); + if (!ok) { + goto end; + } + + // usb_device not needed anymore + sc_usb_device_destroy(usb_device); + usb_device_initialized = false; + + ret = event_loop(s); + LOGD("quit..."); + +end: + if (aoa_started) { + sc_aoa_stop(&s->aoa); + } + sc_usb_stop(&s->usb); + + if (mouse) { + sc_hid_mouse_destroy(&s->mouse); + } + if (keyboard) { + sc_hid_keyboard_destroy(&s->keyboard); + } + + if (aoa_initialized) { + sc_aoa_join(&s->aoa); + sc_aoa_destroy(&s->aoa); + } + + sc_usb_join(&s->usb); + + if (usb_connected) { + sc_usb_disconnect(&s->usb); + } + + if (usb_device_initialized) { + sc_usb_device_destroy(usb_device); + } + + sc_usb_destroy(&s->usb); + + return ret; +} diff --git a/app/src/usb/scrcpy_otg.h b/app/src/usb/scrcpy_otg.h new file mode 100644 index 0000000000..24b9cde87c --- /dev/null +++ b/app/src/usb/scrcpy_otg.h @@ -0,0 +1,12 @@ +#ifndef SCRCPY_OTG_H +#define SCRCPY_OTG_H + +#include "common.h" + +#include +#include "options.h" + +bool +scrcpy_otg(struct scrcpy_options *options); + +#endif diff --git a/app/src/usb/screen_otg.c b/app/src/usb/screen_otg.c new file mode 100644 index 0000000000..a4b3726421 --- /dev/null +++ b/app/src/usb/screen_otg.c @@ -0,0 +1,254 @@ +#include "screen_otg.h" + +#include "icon.h" +#include "options.h" +#include "util/log.h" + +static void +sc_screen_otg_capture_mouse(struct sc_screen_otg *screen, bool capture) { + if (SDL_SetRelativeMouseMode(capture)) { + LOGE("Could not set relative mouse mode to %s: %s", + capture ? "true" : "false", SDL_GetError()); + return; + } + + screen->mouse_captured = capture; +} + +static void +sc_screen_otg_render(struct sc_screen_otg *screen) { + SDL_RenderClear(screen->renderer); + if (screen->texture) { + SDL_RenderCopy(screen->renderer, screen->texture, NULL, NULL); + } + SDL_RenderPresent(screen->renderer); +} + +bool +sc_screen_otg_init(struct sc_screen_otg *screen, + const struct sc_screen_otg_params *params) { + screen->keyboard = params->keyboard; + screen->mouse = params->mouse; + + screen->mouse_captured = false; + screen->mouse_capture_key_pressed = 0; + + const char *title = params->window_title; + assert(title); + + int x = params->window_x != SC_WINDOW_POSITION_UNDEFINED + ? params->window_x : (int) SDL_WINDOWPOS_UNDEFINED; + int y = params->window_y != SC_WINDOW_POSITION_UNDEFINED + ? params->window_y : (int) SDL_WINDOWPOS_UNDEFINED; + int width = 256; + int height = 256; + + uint32_t window_flags = 0; + if (params->always_on_top) { + window_flags |= SDL_WINDOW_ALWAYS_ON_TOP; + } + if (params->window_borderless) { + window_flags |= SDL_WINDOW_BORDERLESS; + } + + screen->window = SDL_CreateWindow(title, x, y, width, height, window_flags); + if (!screen->window) { + LOGE("Could not create window: %s", SDL_GetError()); + return false; + } + + screen->renderer = SDL_CreateRenderer(screen->window, -1, 0); + if (!screen->renderer) { + LOGE("Could not create renderer: %s", SDL_GetError()); + goto error_destroy_window; + } + + SDL_Surface *icon = scrcpy_icon_load(); + + if (icon) { + SDL_SetWindowIcon(screen->window, icon); + + screen->texture = SDL_CreateTextureFromSurface(screen->renderer, icon); + scrcpy_icon_destroy(icon); + if (!screen->texture) { + goto error_destroy_renderer; + } + } else { + screen->texture = NULL; + LOGW("Could not load icon"); + } + + // Capture mouse on start + sc_screen_otg_capture_mouse(screen, true); + + return true; + +error_destroy_window: + SDL_DestroyWindow(screen->window); +error_destroy_renderer: + SDL_DestroyRenderer(screen->renderer); + + return false; +} + +void +sc_screen_otg_destroy(struct sc_screen_otg *screen) { + if (screen->texture) { + SDL_DestroyTexture(screen->texture); + } + SDL_DestroyRenderer(screen->renderer); + SDL_DestroyWindow(screen->window); +} + +static inline bool +sc_screen_otg_is_mouse_capture_key(SDL_Keycode key) { + return key == SDLK_LALT || key == SDLK_LGUI || key == SDLK_RGUI; +} + +static void +sc_screen_otg_process_key(struct sc_screen_otg *screen, + const SDL_KeyboardEvent *event) { + assert(screen->keyboard); + struct sc_key_processor *kp = &screen->keyboard->key_processor; + + struct sc_key_event evt = { + .action = sc_action_from_sdl_keyboard_type(event->type), + .keycode = sc_keycode_from_sdl(event->keysym.sym), + .scancode = sc_scancode_from_sdl(event->keysym.scancode), + .repeat = event->repeat, + .mods_state = sc_mods_state_from_sdl(event->keysym.mod), + }; + + assert(kp->ops->process_key); + kp->ops->process_key(kp, &evt, SC_SEQUENCE_INVALID); +} + +static void +sc_screen_otg_process_mouse_motion(struct sc_screen_otg *screen, + const SDL_MouseMotionEvent *event) { + assert(screen->mouse); + struct sc_mouse_processor *mp = &screen->mouse->mouse_processor; + + struct sc_mouse_motion_event evt = { + // .position not used for HID events + .xrel = event->xrel, + .yrel = event->yrel, + .buttons_state = sc_mouse_buttons_state_from_sdl(event->state, true), + }; + + assert(mp->ops->process_mouse_motion); + mp->ops->process_mouse_motion(mp, &evt); +} + +static void +sc_screen_otg_process_mouse_button(struct sc_screen_otg *screen, + const SDL_MouseButtonEvent *event) { + assert(screen->mouse); + struct sc_mouse_processor *mp = &screen->mouse->mouse_processor; + + uint32_t sdl_buttons_state = SDL_GetMouseState(NULL, NULL); + + struct sc_mouse_click_event evt = { + // .position not used for HID events + .action = sc_action_from_sdl_mousebutton_type(event->type), + .button = sc_mouse_button_from_sdl(event->button), + .buttons_state = + sc_mouse_buttons_state_from_sdl(sdl_buttons_state, true), + }; + + assert(mp->ops->process_mouse_click); + mp->ops->process_mouse_click(mp, &evt); +} + +static void +sc_screen_otg_process_mouse_wheel(struct sc_screen_otg *screen, + const SDL_MouseWheelEvent *event) { + assert(screen->mouse); + struct sc_mouse_processor *mp = &screen->mouse->mouse_processor; + + uint32_t sdl_buttons_state = SDL_GetMouseState(NULL, NULL); + + struct sc_mouse_scroll_event evt = { + // .position not used for HID events + .hscroll = event->x, + .vscroll = event->y, + .buttons_state = + sc_mouse_buttons_state_from_sdl(sdl_buttons_state, true), + }; + + assert(mp->ops->process_mouse_scroll); + mp->ops->process_mouse_scroll(mp, &evt); +} + +void +sc_screen_otg_handle_event(struct sc_screen_otg *screen, SDL_Event *event) { + switch (event->type) { + case SDL_WINDOWEVENT: + switch (event->window.event) { + case SDL_WINDOWEVENT_EXPOSED: + sc_screen_otg_render(screen); + break; + case SDL_WINDOWEVENT_FOCUS_LOST: + sc_screen_otg_capture_mouse(screen, false); + break; + } + return; + case SDL_KEYDOWN: { + SDL_Keycode key = event->key.keysym.sym; + if (sc_screen_otg_is_mouse_capture_key(key)) { + if (!screen->mouse_capture_key_pressed) { + screen->mouse_capture_key_pressed = key; + } else { + // Another mouse capture key has been pressed, cancel mouse + // (un)capture + screen->mouse_capture_key_pressed = 0; + } + // Mouse capture keys are never forwarded to the device + return; + } + + sc_screen_otg_process_key(screen, &event->key); + break; + } + case SDL_KEYUP: { + SDL_Keycode key = event->key.keysym.sym; + SDL_Keycode cap = screen->mouse_capture_key_pressed; + screen->mouse_capture_key_pressed = 0; + if (sc_screen_otg_is_mouse_capture_key(key)) { + if (key == cap) { + // A mouse capture key has been pressed then released: + // toggle the capture mouse mode + sc_screen_otg_capture_mouse(screen, + !screen->mouse_captured); + } + // Mouse capture keys are never forwarded to the device + return; + } + + sc_screen_otg_process_key(screen, &event->key); + break; + } + case SDL_MOUSEMOTION: + if (screen->mouse_captured) { + sc_screen_otg_process_mouse_motion(screen, &event->motion); + } + break; + case SDL_MOUSEBUTTONDOWN: + if (screen->mouse_captured) { + sc_screen_otg_process_mouse_button(screen, &event->button); + } + break; + case SDL_MOUSEBUTTONUP: + if (screen->mouse_captured) { + sc_screen_otg_process_mouse_button(screen, &event->button); + } else { + sc_screen_otg_capture_mouse(screen, true); + } + break; + case SDL_MOUSEWHEEL: + if (screen->mouse_captured) { + sc_screen_otg_process_mouse_wheel(screen, &event->wheel); + } + break; + } +} diff --git a/app/src/usb/screen_otg.h b/app/src/usb/screen_otg.h new file mode 100644 index 0000000000..3fa1c4ad2a --- /dev/null +++ b/app/src/usb/screen_otg.h @@ -0,0 +1,46 @@ +#ifndef SC_SCREEN_OTG_H +#define SC_SCREEN_OTG_H + +#include "common.h" + +#include +#include + +#include "hid_keyboard.h" +#include "hid_mouse.h" + +struct sc_screen_otg { + struct sc_hid_keyboard *keyboard; + struct sc_hid_mouse *mouse; + + SDL_Window *window; + SDL_Renderer *renderer; + SDL_Texture *texture; + + // See equivalent mechanism in screen.h + bool mouse_captured; + SDL_Keycode mouse_capture_key_pressed; +}; + +struct sc_screen_otg_params { + struct sc_hid_keyboard *keyboard; + struct sc_hid_mouse *mouse; + + const char *window_title; + bool always_on_top; + int16_t window_x; // accepts SC_WINDOW_POSITION_UNDEFINED + int16_t window_y; // accepts SC_WINDOW_POSITION_UNDEFINED + bool window_borderless; +}; + +bool +sc_screen_otg_init(struct sc_screen_otg *screen, + const struct sc_screen_otg_params *params); + +void +sc_screen_otg_destroy(struct sc_screen_otg *screen); + +void +sc_screen_otg_handle_event(struct sc_screen_otg *screen, SDL_Event *event); + +#endif