diff --git a/app/meson.build b/app/meson.build index 3a5cb12ade..d6fac58048 100644 --- a/app/meson.build +++ b/app/meson.build @@ -25,6 +25,7 @@ src = [ 'src/server.c', 'src/stream.c', 'src/video_buffer.c', + 'src/util/acksync.c', 'src/util/file.c', 'src/util/intr.c', 'src/util/log.c', diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c index 4c0b2bdaa0..6fd32610cb 100644 --- a/app/src/aoa_hid.c +++ b/app/src/aoa_hid.c @@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, hid_event->accessory_id = accessory_id; hid_event->buffer = buffer; hid_event->size = buffer_size; - hid_event->delay = 0; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; } void @@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { } bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial) { +sc_aoa_init(struct sc_aoa *aoa, const char *serial, + struct sc_acksync *acksync) { + assert(acksync); + cbuf_init(&aoa->queue); if (!sc_mutex_init(&aoa->mutex)) { @@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) { } aoa->stopped = false; + aoa->acksync = acksync; return true; } @@ -332,23 +336,28 @@ run_aoa_thread(void *data) { assert(non_empty); (void) non_empty; - assert(event.delay >= 0); - if (event.delay) { - // Wait during the specified delay before injecting the HID event - sc_tick deadline = sc_tick_now() + event.delay; - bool timed_out = false; - while (!aoa->stopped && !timed_out) { - timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex, - deadline); - } - if (aoa->stopped) { - sc_mutex_unlock(&aoa->mutex); + uint64_t ack_to_wait = event.ack_to_wait; + sc_mutex_unlock(&aoa->mutex); + + if (ack_to_wait != SC_SEQUENCE_INVALID) { + LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait); + // Do not block the loop indefinitely if the ack never comes (it should + // never happen) + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500); + enum sc_acksync_wait_result result = + sc_acksync_wait(aoa->acksync, ack_to_wait, deadline); + + if (result == SC_ACKSYNC_WAIT_TIMEOUT) { + LOGW("Ack not received after 500ms, discarding HID event"); + sc_hid_event_destroy(&event); + continue; + } else if (result == SC_ACKSYNC_WAIT_INTR) { + // stopped + sc_hid_event_destroy(&event); break; } } - sc_mutex_unlock(&aoa->mutex); - bool ok = sc_aoa_send_hid_event(aoa, &event); sc_hid_event_destroy(&event); if (!ok) { @@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) { aoa->stopped = true; sc_cond_signal(&aoa->event_cond); sc_mutex_unlock(&aoa->mutex); + + sc_acksync_interrupt(aoa->acksync); } void diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h index 24cef50212..e8fb970822 100644 --- a/app/src/aoa_hid.h +++ b/app/src/aoa_hid.h @@ -6,6 +6,7 @@ #include +#include "util/acksync.h" #include "util/cbuf.h" #include "util/thread.h" #include "util/tick.h" @@ -14,7 +15,7 @@ struct sc_hid_event { uint16_t accessory_id; unsigned char *buffer; uint16_t size; - sc_tick delay; + uint64_t ack_to_wait; }; // Takes ownership of buffer @@ -36,10 +37,12 @@ struct sc_aoa { sc_cond event_cond; bool stopped; struct sc_hid_event_queue queue; + + struct sc_acksync *acksync; }; bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial); +sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync); void sc_aoa_destroy(struct sc_aoa *aoa); diff --git a/app/src/compat.h b/app/src/compat.h index 32759c01b4..c06c23fae2 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -6,6 +6,12 @@ #include #include +#ifndef __WIN32 +# define PRIu64_ PRIu64 +#else +# define PRIu64_ "I64u" // Windows... +#endif + // In ffmpeg/doc/APIchanges: // 2018-02-06 - 0694d87024 - lavf 58.9.100 - avformat.h // Deprecate use of av_register_input_format(), av_register_output_format(), diff --git a/app/src/control_msg.c b/app/src/control_msg.c index 74e3315c52..90cde0cfa2 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -118,11 +118,12 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) { buf[1] = msg->inject_keycode.action; return 2; case CONTROL_MSG_TYPE_SET_CLIPBOARD: { - buf[1] = !!msg->set_clipboard.paste; + buffer_write64be(&buf[1], msg->set_clipboard.sequence); + buf[9] = !!msg->set_clipboard.paste; size_t len = write_string(msg->set_clipboard.text, CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH, - &buf[2]); - return 2 + len; + &buf[10]); + return 10 + len; } case CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE: buf[1] = msg->set_screen_power_mode.mode; @@ -170,11 +171,6 @@ control_msg_log(const struct control_msg *msg) { (long) msg->inject_touch_event.buttons); } else { // numeric pointer id -#ifndef __WIN32 -# define PRIu64_ PRIu64 -#else -# define PRIu64_ "I64u" // Windows... -#endif LOG_CMSG("touch [id=%" PRIu64_ "] %-4s position=%" PRIi32 ",%" PRIi32 " pressure=%g buttons=%06lx", id, @@ -199,7 +195,8 @@ control_msg_log(const struct control_msg *msg) { KEYEVENT_ACTION_LABEL(msg->inject_keycode.action)); break; case CONTROL_MSG_TYPE_SET_CLIPBOARD: - LOG_CMSG("clipboard %s \"%s\"", + LOG_CMSG("clipboard %" PRIu64_ " %s \"%s\"", + msg->set_clipboard.sequence, msg->set_clipboard.paste ? "paste" : "copy", msg->set_clipboard.text); break; diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 16492849ab..7352defe7b 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -70,6 +70,7 @@ struct control_msg { // screen may only be turned on on ACTION_DOWN } back_or_screen_on; struct { + uint64_t sequence; char *text; // owned, to be freed by free() bool paste; } set_clipboard; diff --git a/app/src/controller.c b/app/src/controller.c index e486ea7224..6cf3d20e47 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -5,10 +5,11 @@ #include "util/log.h" bool -controller_init(struct controller *controller, sc_socket control_socket) { +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync) { cbuf_init(&controller->queue); - bool ok = receiver_init(&controller->receiver, control_socket); + bool ok = receiver_init(&controller->receiver, control_socket, acksync); if (!ok) { return false; } diff --git a/app/src/controller.h b/app/src/controller.h index e700413100..00267878ac 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -7,6 +7,7 @@ #include "control_msg.h" #include "receiver.h" +#include "util/acksync.h" #include "util/cbuf.h" #include "util/net.h" #include "util/thread.h" @@ -24,7 +25,8 @@ struct controller { }; bool -controller_init(struct controller *controller, sc_socket control_socket); +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync); void controller_destroy(struct controller *controller); diff --git a/app/src/device_msg.c b/app/src/device_msg.c index 827f421379..4163b9fc9f 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -1,5 +1,6 @@ #include "device_msg.h" +#include #include #include @@ -34,6 +35,11 @@ device_msg_deserialize(const unsigned char *buf, size_t len, msg->clipboard.text = text; return 5 + clipboard_len; } + case DEVICE_MSG_TYPE_ACK_CLIPBOARD: { + uint64_t sequence = buffer_read64be(&buf[1]); + msg->ack_clipboard.sequence = sequence; + return 9; + } default: LOGW("Unknown device message type: %d", (int) msg->type); return -1; // error, we cannot recover diff --git a/app/src/device_msg.h b/app/src/device_msg.h index 888d9216ad..a0c989e3f8 100644 --- a/app/src/device_msg.h +++ b/app/src/device_msg.h @@ -13,6 +13,7 @@ enum device_msg_type { DEVICE_MSG_TYPE_CLIPBOARD, + DEVICE_MSG_TYPE_ACK_CLIPBOARD, }; struct device_msg { @@ -21,6 +22,9 @@ struct device_msg { struct { char *text; // owned, to be freed by free() } clipboard; + struct { + uint64_t sequence; + } ack_clipboard; }; }; diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c index 3ac1a4417e..2afc09b2f4 100644 --- a/app/src/hid_keyboard.c +++ b/app/src/hid_keyboard.c @@ -278,7 +278,8 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) { static void sc_key_processor_process_key(struct sc_key_processor *kp, - const SDL_KeyboardEvent *event) { + const SDL_KeyboardEvent *event, + uint64_t ack_to_wait) { if (event->repeat) { // In USB HID protocol, key repeat is handled by the host (Android), so // just ignore key repeat here. @@ -298,16 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - SDL_Keycode keycode = event->keysym.sym; - bool down = event->type == SDL_KEYDOWN; - bool ctrl = event->keysym.mod & KMOD_CTRL; - bool shift = event->keysym.mod & KMOD_SHIFT; - if (ctrl && !shift && keycode == SDLK_v && down) { + if (ack_to_wait) { // Ctrl+v is pressed, so clipboard synchronization has been - // requested. Wait a bit so that the clipboard is set before - // injecting Ctrl+v via HID, otherwise it would paste the old - // clipboard content. - hid_event.delay = SC_TICK_FROM_MS(5); + // requested. Wait until clipboard synchronization is acknowledged + // by the server, otherwise it could paste the old clipboard + // content. + hid_event.ack_to_wait = ack_to_wait; } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { @@ -348,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { .process_text = sc_key_processor_process_text, }; + // Clipboard synchronization is requested over the control socket, while HID + // events are sent over AOA, so it must wait for clipboard synchronization + // to be acknowledged by the device before injecting Ctrl+v. + kb->key_processor.async_paste = true; kb->key_processor.ops = &ops; return true; diff --git a/app/src/input_manager.c b/app/src/input_manager.c index b84f3bea60..f1715b79f0 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller, im->last_keycode = SDLK_UNKNOWN; im->last_mod = 0; im->key_repeat = 0; + + im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID } static void @@ -208,35 +210,35 @@ collapse_panels(struct controller *controller) { } } -static void -set_device_clipboard(struct controller *controller, bool paste) { +static bool +set_device_clipboard(struct controller *controller, bool paste, + uint64_t sequence) { char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); - return; - } - if (!*text) { - // empty text - SDL_free(text); - return; + return false; } char *text_dup = strdup(text); SDL_free(text); if (!text_dup) { LOGW("Could not strdup input text"); - return; + return false; } struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD; + msg.set_clipboard.sequence = sequence; msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; if (!controller_push_msg(controller, &msg)) { free(text_dup); LOGW("Could not request 'set device clipboard'"); + return false; } + + return true; } static void @@ -462,8 +464,10 @@ input_manager_process_key(struct input_manager *im, // inject the text as input events clipboard_paste(controller); } else { - // store the text in the device clipboard and paste - set_device_clipboard(controller, true); + // store the text in the device clipboard and paste, + // without requesting an acknowledgment + set_device_clipboard(controller, true, + SC_SEQUENCE_INVALID); } } return; @@ -512,18 +516,36 @@ input_manager_process_key(struct input_manager *im, return; } - if (ctrl && !shift && keycode == SDLK_v && down && !repeat) { + uint64_t ack_to_wait = SC_SEQUENCE_INVALID; + bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat; + if (is_ctrl_v) { if (im->legacy_paste) { // inject the text as input events clipboard_paste(controller); return; } + + // Request an acknowledgement only if necessary + uint64_t sequence = im->kp->async_paste ? im->next_sequence + : SC_SEQUENCE_INVALID; + // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - set_device_clipboard(controller, false); + bool ok = set_device_clipboard(controller, false, sequence); + if (!ok) { + LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); + return; + } + + if (im->kp->async_paste) { + // The key processor must wait for this ack before injecting Ctrl+v + ack_to_wait = sequence; + // Increment only when the request succeeded + ++im->next_sequence; + } } - im->kp->ops->process_key(im->kp, event); + im->kp->ops->process_key(im->kp, event, ack_to_wait); } static void diff --git a/app/src/input_manager.h b/app/src/input_manager.h index f018f98a85..22d773811c 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -38,6 +38,8 @@ struct input_manager { unsigned key_repeat; SDL_Keycode last_keycode; uint16_t last_mod; + + uint64_t next_sequence; // used for request acknowledgements }; void diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c index bcc85da8e6..4112fd4cb7 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_inject.c @@ -188,7 +188,13 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, static void sc_key_processor_process_key(struct sc_key_processor *kp, - const SDL_KeyboardEvent *event) { + const SDL_KeyboardEvent *event, + uint64_t ack_to_wait) { + // The device clipboard synchronization and the key event messages are + // serialized, there is nothing special to do to ensure that the clipboard + // is set before injecting Ctrl+v. + (void) ack_to_wait; + struct sc_keyboard_inject *ki = DOWNCAST(kp); if (event->repeat) { @@ -250,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki, .process_text = sc_key_processor_process_text, }; + // Key injection and clipboard synchronization are serialized + ki->key_processor.async_paste = false; ki->key_processor.ops = &ops; } diff --git a/app/src/receiver.c b/app/src/receiver.c index b5cf9b395b..eeb206f173 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -7,12 +7,16 @@ #include "util/log.h" bool -receiver_init(struct receiver *receiver, sc_socket control_socket) { +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync) { bool ok = sc_mutex_init(&receiver->mutex); if (!ok) { return false; } + receiver->control_socket = control_socket; + receiver->acksync = acksync; + return true; } @@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) { } static void -process_msg(struct device_msg *msg) { +process_msg(struct receiver *receiver, struct device_msg *msg) { switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { char *current = SDL_GetClipboardText(); @@ -37,11 +41,17 @@ process_msg(struct device_msg *msg) { SDL_SetClipboardText(msg->clipboard.text); break; } + case DEVICE_MSG_TYPE_ACK_CLIPBOARD: + assert(receiver->acksync); + LOGD("Ack device clipboard sequence=%" PRIu64_, + msg->ack_clipboard.sequence); + sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); + break; } } static ssize_t -process_msgs(const unsigned char *buf, size_t len) { +process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) { size_t head = 0; for (;;) { struct device_msg msg; @@ -53,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) { return head; } - process_msg(&msg); + process_msg(receiver, &msg); device_msg_destroy(&msg); head += r; @@ -81,7 +91,7 @@ run_receiver(void *data) { } head += r; - ssize_t consumed = process_msgs(buf, head); + ssize_t consumed = process_msgs(receiver, buf, head); if (consumed == -1) { // an error occurred break; diff --git a/app/src/receiver.h b/app/src/receiver.h index 99f128a4be..3c4e8c645a 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -5,6 +5,7 @@ #include +#include "util/acksync.h" #include "util/net.h" #include "util/thread.h" @@ -14,10 +15,13 @@ struct receiver { sc_socket control_socket; sc_thread thread; sc_mutex mutex; + + struct sc_acksync *acksync; }; bool -receiver_init(struct receiver *receiver, sc_socket control_socket); +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync); void receiver_destroy(struct receiver *receiver); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 61087681c6..04239bf540 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,7 @@ #include "screen.h" #include "server.h" #include "stream.h" +#include "util/acksync.h" #include "util/log.h" #include "util/net.h" #ifdef HAVE_V4L2 @@ -46,6 +47,8 @@ struct scrcpy { struct file_handler file_handler; #ifdef HAVE_AOA_HID struct sc_aoa aoa; + // sequence/ack helper to synchronize clipboard and Ctrl+v via HID + struct sc_acksync acksync; #endif union { struct sc_keyboard_inject keyboard_inject; @@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) { bool controller_started = false; bool screen_initialized = false; + struct sc_acksync *acksync = NULL; + struct sc_server_params params = { .serial = options->serial, .log_level = options->log_level, @@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) { } if (options->control) { - if (!controller_init(&s->controller, s->server.control_socket)) { +#ifdef HAVE_AOA_HID + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) { + bool ok = sc_acksync_init(&s->acksync); + if (!ok) { + goto end; + } + + acksync = &s->acksync; + } +#endif + if (!controller_init(&s->controller, s->server.control_socket, + acksync)) { goto end; } controller_initialized = true; @@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool aoa_hid_ok = false; - bool ok = sc_aoa_init(&s->aoa, serial); + bool ok = sc_aoa_init(&s->aoa, serial, acksync); if (!ok) { goto aoa_hid_end; } @@ -586,6 +602,9 @@ scrcpy(struct scrcpy_options *options) { sc_hid_keyboard_destroy(&s->keyboard_hid); sc_aoa_stop(&s->aoa); } + if (acksync) { + sc_acksync_destroy(acksync); + } #endif if (controller_started) { controller_stop(&s->controller); diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 5790310b79..f4afe27b17 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -14,12 +14,31 @@ * Component able to process and inject keys should implement this trait. */ struct sc_key_processor { + /** + * Set by the implementation to indicate that it must explicitly wait for + * the clipboard to be set on the device before injecting Ctrl+v to avoid + * race conditions. If it is set, the input_manager will pass a valid + * ack_to_wait to process_key() in case of clipboard synchronization + * resulting of the key event. + */ + bool async_paste; + const struct sc_key_processor_ops *ops; }; struct sc_key_processor_ops { + + /** + * Process the keyboard event + * + * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates + * the acknowledgement number to wait for before injecting this event. + * This allows to ensure that the device clipboard is set before injecting + * Ctrl+v on the device. + */ void - (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event); + (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, + uint64_t ack_to_wait); void (*process_text)(struct sc_key_processor *kp, diff --git a/app/src/util/acksync.c b/app/src/util/acksync.c new file mode 100644 index 0000000000..2899cdcbe0 --- /dev/null +++ b/app/src/util/acksync.c @@ -0,0 +1,76 @@ +#include "acksync.h" + +#include +#include "util/log.h" + +bool +sc_acksync_init(struct sc_acksync *as) { + bool ok = sc_mutex_init(&as->mutex); + if (!ok) { + return false; + } + + ok = sc_cond_init(&as->cond); + if (!ok) { + sc_mutex_destroy(&as->mutex); + return false; + } + + as->stopped = false; + as->ack = SC_SEQUENCE_INVALID; + + return true; +} + +void +sc_acksync_destroy(struct sc_acksync *as) { + sc_cond_destroy(&as->cond); + sc_mutex_destroy(&as->mutex); +} + +void +sc_acksync_ack(struct sc_acksync *as, uint64_t sequence) { + sc_mutex_lock(&as->mutex); + + // Acknowledgements must be monotonic + assert(sequence >= as->ack); + + as->ack = sequence; + sc_cond_signal(&as->cond); + + sc_mutex_unlock(&as->mutex); +} + +enum sc_acksync_wait_result +sc_acksync_wait(struct sc_acksync *as, uint64_t ack, sc_tick deadline) { + sc_mutex_lock(&as->mutex); + + bool timed_out = false; + while (!as->stopped && as->ack < ack && !timed_out) { + timed_out = !sc_cond_timedwait(&as->cond, &as->mutex, deadline); + } + + enum sc_acksync_wait_result ret; + if (as->stopped) { + ret = SC_ACKSYNC_WAIT_INTR; + } else if (as->ack >= ack) { + ret = SC_ACKSYNC_WAIT_OK; + } else { + assert(timed_out); + ret = SC_ACKSYNC_WAIT_TIMEOUT; + } + sc_mutex_unlock(&as->mutex); + + return ret; +} + +/** + * Interrupt any `sc_acksync_wait()` + */ +void +sc_acksync_interrupt(struct sc_acksync *as) { + sc_mutex_lock(&as->mutex); + as->stopped = true; + sc_cond_signal(&as->cond); + sc_mutex_unlock(&as->mutex); +} diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h new file mode 100644 index 0000000000..58ab1b353c --- /dev/null +++ b/app/src/util/acksync.h @@ -0,0 +1,66 @@ +#ifndef SC_ACK_SYNC_H +#define SC_ACK_SYNC_H + +#include "common.h" + +#include "thread.h" + +#define SC_SEQUENCE_INVALID 0 + +/** + * Helper to wait for acknowledgments + * + * In practice, it is used to wait for device clipboard acknowledgement from the + * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the + * content of the old device clipboard (if Ctrl+v was injected before the + * clipboard content was actually set). + */ +struct sc_acksync { + sc_mutex mutex; + sc_cond cond; + + bool stopped; + + // Last acked value, initially SC_SEQUENCE_INVALID + uint64_t ack; +}; + +enum sc_acksync_wait_result { + // Acknowledgment received + SC_ACKSYNC_WAIT_OK, + + // Timeout expired + SC_ACKSYNC_WAIT_TIMEOUT, + + // Interrupted from another thread by sc_acksync_interrupt() + SC_ACKSYNC_WAIT_INTR, +}; + +bool +sc_acksync_init(struct sc_acksync *as); + +void +sc_acksync_destroy(struct sc_acksync *as); + +/** + * Acknowledge `sequence` + * + * The `sequence` must be greater than (or equal to) any previous acknowledged + * sequence. + */ +void +sc_acksync_ack(struct sc_acksync *as, uint64_t sequence); + +/** + * Wait for acknowledgment of sequence `ack` (or higher) + */ +enum sc_acksync_wait_result +sc_acksync_wait(struct sc_acksync *as, uint64_t ack, sc_tick deadline); + +/** + * Interrupt any `sc_acksync_wait()` + */ +void +sc_acksync_interrupt(struct sc_acksync *as); + +#endif diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index b237e28e4d..5cd7056b22 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -226,6 +226,7 @@ static void test_serialize_set_clipboard(void) { struct control_msg msg = { .type = CONTROL_MSG_TYPE_SET_CLIPBOARD, .set_clipboard = { + .sequence = UINT64_C(0x0102030405060708), .paste = true, .text = "hello, world!", }, @@ -233,10 +234,11 @@ static void test_serialize_set_clipboard(void) { unsigned char buf[CONTROL_MSG_MAX_SIZE]; size_t size = control_msg_serialize(&msg, buf); - assert(size == 19); + assert(size == 27); const unsigned char expected[] = { CONTROL_MSG_TYPE_SET_CLIPBOARD, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence 1, // paste 0x00, 0x00, 0x00, 0x0d, // text length 'h', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // text diff --git a/app/tests/test_device_msg_deserialize.c b/app/tests/test_device_msg_deserialize.c index 3427d64045..835096c000 100644 --- a/app/tests/test_device_msg_deserialize.c +++ b/app/tests/test_device_msg_deserialize.c @@ -47,11 +47,26 @@ static void test_deserialize_clipboard_big(void) { device_msg_destroy(&msg); } +static void test_deserialize_ack_set_clipboard(void) { + const unsigned char input[] = { + DEVICE_MSG_TYPE_ACK_CLIPBOARD, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence + }; + + struct device_msg msg; + ssize_t r = device_msg_deserialize(input, sizeof(input), &msg); + assert(r == 9); + + assert(msg.type == DEVICE_MSG_TYPE_ACK_CLIPBOARD); + assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708)); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; test_deserialize_clipboard(); test_deserialize_clipboard_big(); + test_deserialize_ack_set_clipboard(); return 0; } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index f8edd53c11..2cd8019197 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -18,6 +18,8 @@ public final class ControlMessage { public static final int TYPE_SET_SCREEN_POWER_MODE = 10; public static final int TYPE_ROTATE_DEVICE = 11; + public static final long SEQUENCE_INVALID = 0; + private int type; private String text; private int metaState; // KeyEvent.META_* @@ -31,6 +33,7 @@ public final class ControlMessage { private int vScroll; private boolean paste; private int repeat; + private long sequence; private ControlMessage() { } @@ -79,9 +82,10 @@ public static ControlMessage createBackOrScreenOn(int action) { return msg; } - public static ControlMessage createSetClipboard(String text, boolean paste) { + public static ControlMessage createSetClipboard(long sequence, String text, boolean paste) { ControlMessage msg = new ControlMessage(); msg.type = TYPE_SET_CLIPBOARD; + msg.sequence = sequence; msg.text = text; msg.paste = paste; return msg; @@ -154,4 +158,8 @@ public boolean getPaste() { public int getRepeat() { return repeat; } + + public long getSequence() { + return sequence; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index e4ab840201..80931e9426 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -13,11 +13,11 @@ public class ControlMessageReader { static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20; static final int BACK_OR_SCREEN_ON_LENGTH = 1; static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1; - static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 1; + static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 9; private static final int MESSAGE_MAX_SIZE = 1 << 18; // 256k - public static final int CLIPBOARD_TEXT_MAX_LENGTH = MESSAGE_MAX_SIZE - 6; // type: 1 byte; paste flag: 1 byte; length: 4 bytes + public static final int CLIPBOARD_TEXT_MAX_LENGTH = MESSAGE_MAX_SIZE - 14; // type: 1 byte; sequence: 8 bytes; paste flag: 1 byte; length: 4 bytes public static final int INJECT_TEXT_MAX_LENGTH = 300; private final byte[] rawBuffer = new byte[MESSAGE_MAX_SIZE]; @@ -166,12 +166,13 @@ private ControlMessage parseSetClipboard() { if (buffer.remaining() < SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH) { return null; } + long sequence = buffer.getLong(); boolean paste = buffer.get() != 0; String text = parseString(); if (text == null) { return null; } - return ControlMessage.createSetClipboard(text, paste); + return ControlMessage.createSetClipboard(sequence, text, paste); } private ControlMessage parseSetScreenPowerMode() { diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 45882bb9c9..8b24b300cd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -120,7 +120,12 @@ private void handleEvent() throws IOException { } break; case ControlMessage.TYPE_SET_CLIPBOARD: + long sequence = msg.getSequence(); setClipboard(msg.getText(), msg.getPaste()); + if (sequence != ControlMessage.SEQUENCE_INVALID) { + // Acknowledgement requested + sender.pushAckClipboard(sequence); + } break; case ControlMessage.TYPE_SET_SCREEN_POWER_MODE: if (device.supportsInputEvents()) { diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java index c6eebd3806..5b7c4de5b5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java @@ -3,9 +3,13 @@ public final class DeviceMessage { public static final int TYPE_CLIPBOARD = 0; + public static final int TYPE_ACK_CLIPBOARD = 1; + + public static final long SEQUENCE_INVALID = ControlMessage.SEQUENCE_INVALID; private int type; private String text; + private long sequence; private DeviceMessage() { } @@ -17,6 +21,13 @@ public static DeviceMessage createClipboard(String text) { return event; } + public static DeviceMessage createAckClipboard(long sequence) { + DeviceMessage event = new DeviceMessage(); + event.type = TYPE_ACK_CLIPBOARD; + event.sequence = sequence; + return event; + } + public int getType() { return type; } @@ -24,4 +35,8 @@ public int getType() { public String getText() { return text; } + + public long getSequence() { + return sequence; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java index bbf4dd2eef..4ebccaccf6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java @@ -8,6 +8,8 @@ public final class DeviceMessageSender { private String clipboardText; + private long ack; + public DeviceMessageSender(DesktopConnection connection) { this.connection = connection; } @@ -17,18 +19,34 @@ public synchronized void pushClipboardText(String text) { notify(); } + public synchronized void pushAckClipboard(long sequence) { + ack = sequence; + notify(); + } + public void loop() throws IOException, InterruptedException { while (true) { String text; + long sequence; synchronized (this) { - while (clipboardText == null) { + while (ack == DeviceMessage.SEQUENCE_INVALID && clipboardText == null) { wait(); } text = clipboardText; clipboardText = null; + + sequence = ack; + ack = DeviceMessage.SEQUENCE_INVALID; + } + + if (sequence != DeviceMessage.SEQUENCE_INVALID) { + DeviceMessage event = DeviceMessage.createAckClipboard(sequence); + connection.sendDeviceMessage(event); + } + if (text != null) { + DeviceMessage event = DeviceMessage.createClipboard(text); + connection.sendDeviceMessage(event); } - DeviceMessage event = DeviceMessage.createClipboard(text); - connection.sendDeviceMessage(event); } } } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java index 15d91a35b7..bcd8d20676 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java @@ -15,7 +15,7 @@ public class DeviceMessageWriter { public void writeTo(DeviceMessage msg, OutputStream output) throws IOException { buffer.clear(); - buffer.put((byte) DeviceMessage.TYPE_CLIPBOARD); + buffer.put((byte) msg.getType()); switch (msg.getType()) { case DeviceMessage.TYPE_CLIPBOARD: String text = msg.getText(); @@ -25,6 +25,10 @@ public void writeTo(DeviceMessage msg, OutputStream output) throws IOException { buffer.put(raw, 0, len); output.write(rawBuffer, 0, buffer.position()); break; + case DeviceMessage.TYPE_ACK_CLIPBOARD: + buffer.putLong(msg.getSequence()); + output.write(rawBuffer, 0, buffer.position()); + break; default: Ln.w("Unknown device message: " + msg.getType()); break; diff --git a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java index 7f3d3f61fa..3b0b6c0da8 100644 --- a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java @@ -235,6 +235,7 @@ public void testParseSetClipboardEvent() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); dos.writeByte(ControlMessage.TYPE_SET_CLIPBOARD); + dos.writeLong(0x0102030405060708L); // sequence dos.writeByte(1); // paste byte[] text = "testé".getBytes(StandardCharsets.UTF_8); dos.writeInt(text.length); @@ -246,6 +247,7 @@ public void testParseSetClipboardEvent() throws IOException { ControlMessage event = reader.next(); Assert.assertEquals(ControlMessage.TYPE_SET_CLIPBOARD, event.getType()); + Assert.assertEquals(0x0102030405060708L, event.getSequence()); Assert.assertEquals("testé", event.getText()); Assert.assertTrue(event.getPaste()); } @@ -259,6 +261,7 @@ public void testParseBigSetClipboardEvent() throws IOException { dos.writeByte(ControlMessage.TYPE_SET_CLIPBOARD); byte[] rawText = new byte[ControlMessageReader.CLIPBOARD_TEXT_MAX_LENGTH]; + dos.writeLong(0x0807060504030201L); // sequence dos.writeByte(1); // paste Arrays.fill(rawText, (byte) 'a'); String text = new String(rawText, 0, rawText.length); @@ -272,6 +275,7 @@ public void testParseBigSetClipboardEvent() throws IOException { ControlMessage event = reader.next(); Assert.assertEquals(ControlMessage.TYPE_SET_CLIPBOARD, event.getType()); + Assert.assertEquals(0x0807060504030201L, event.getSequence()); Assert.assertEquals(text, event.getText()); Assert.assertTrue(event.getPaste()); } diff --git a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java index 88bf2af9fa..7b917d337c 100644 --- a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java @@ -32,4 +32,24 @@ public void testSerializeClipboard() throws IOException { Assert.assertArrayEquals(expected, actual); } + + @Test + public void testSerializeAckSetClipboard() throws IOException { + DeviceMessageWriter writer = new DeviceMessageWriter(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(DeviceMessage.TYPE_ACK_CLIPBOARD); + dos.writeLong(0x0102030405060708L); + + byte[] expected = bos.toByteArray(); + + DeviceMessage msg = DeviceMessage.createAckClipboard(0x0102030405060708L); + bos = new ByteArrayOutputStream(); + writer.writeTo(msg, bos); + + byte[] actual = bos.toByteArray(); + + Assert.assertArrayEquals(expected, actual); + } }