From ea8028332ced7ef61e755742ad9e91586e3193e3 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 17:39:19 +0100 Subject: [PATCH 1/9] Synchronize computer-to-device empty clipboard Set the device clipboard to empty string if necessary. Otherwise, the current device clipboard will be pasted on Ctrl+v. PR #2814 --- app/src/input_manager.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/src/input_manager.c b/app/src/input_manager.c index b84f3bea60..6158f6d2c0 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -215,11 +215,6 @@ set_device_clipboard(struct controller *controller, bool paste) { LOGW("Could not get clipboard text: %s", SDL_GetError()); return; } - if (!*text) { - // empty text - SDL_free(text); - return; - } char *text_dup = strdup(text); SDL_free(text); From 854de9659a503291fc310ed6702bd4278bf2a725 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 17:44:00 +0100 Subject: [PATCH 2/9] Do not inject Ctrl+v if clipboard sync failed This prevents to paste the current Android clipboard, which would be unexpected. PR #2814 --- app/src/input_manager.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 6158f6d2c0..c15e0427c7 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -208,19 +208,19 @@ collapse_panels(struct controller *controller) { } } -static void +static bool set_device_clipboard(struct controller *controller, bool paste) { char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); - return; + return false; } char *text_dup = strdup(text); SDL_free(text); if (!text_dup) { LOGW("Could not strdup input text"); - return; + return false; } struct control_msg msg; @@ -231,7 +231,10 @@ set_device_clipboard(struct controller *controller, bool paste) { if (!controller_push_msg(controller, &msg)) { free(text_dup); LOGW("Could not request 'set device clipboard'"); + return false; } + + return true; } static void @@ -515,7 +518,11 @@ input_manager_process_key(struct input_manager *im, } // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - set_device_clipboard(controller, false); + bool ok = set_device_clipboard(controller, false); + if (!ok) { + LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); + return; + } } im->kp->ops->process_key(im->kp, event); From 5b3856c3b6bed67abfd20585daa8eb43fa863582 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 21 Nov 2021 17:24:34 +0100 Subject: [PATCH 3/9] Explicitly indicate when device clipboard is set Pass the information that device clipboard has been set to the key processor. This avoids the keyprocessor to "guess", and paves the way to implement a proper acknowledgement mechanism. PR #2814 --- app/src/hid_keyboard.c | 9 +++------ app/src/input_manager.c | 5 +++-- app/src/keyboard_inject.c | 8 +++++++- app/src/trait/key_processor.h | 11 ++++++++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c index 3ac1a4417e..2809ccd6bf 100644 --- a/app/src/hid_keyboard.c +++ b/app/src/hid_keyboard.c @@ -278,7 +278,8 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) { static void sc_key_processor_process_key(struct sc_key_processor *kp, - const SDL_KeyboardEvent *event) { + const SDL_KeyboardEvent *event, + bool device_clipboard_set) { if (event->repeat) { // In USB HID protocol, key repeat is handled by the host (Android), so // just ignore key repeat here. @@ -298,11 +299,7 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - SDL_Keycode keycode = event->keysym.sym; - bool down = event->type == SDL_KEYDOWN; - bool ctrl = event->keysym.mod & KMOD_CTRL; - bool shift = event->keysym.mod & KMOD_SHIFT; - if (ctrl && !shift && keycode == SDLK_v && down) { + if (device_clipboard_set) { // Ctrl+v is pressed, so clipboard synchronization has been // requested. Wait a bit so that the clipboard is set before // injecting Ctrl+v via HID, otherwise it would paste the old diff --git a/app/src/input_manager.c b/app/src/input_manager.c index c15e0427c7..16c822344b 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -510,7 +510,8 @@ input_manager_process_key(struct input_manager *im, return; } - if (ctrl && !shift && keycode == SDLK_v && down && !repeat) { + bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat; + if (is_ctrl_v) { if (im->legacy_paste) { // inject the text as input events clipboard_paste(controller); @@ -525,7 +526,7 @@ input_manager_process_key(struct input_manager *im, } } - im->kp->ops->process_key(im->kp, event); + im->kp->ops->process_key(im->kp, event, is_ctrl_v); } static void diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c index bcc85da8e6..e59b5520e9 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_inject.c @@ -188,7 +188,13 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, static void sc_key_processor_process_key(struct sc_key_processor *kp, - const SDL_KeyboardEvent *event) { + const SDL_KeyboardEvent *event, + bool device_clipboard_set) { + // The device clipboard synchronization and the key event messages are + // serialized, there is nothing special to do to ensure that the clipboard + // is set before injecting Ctrl+v. + (void) device_clipboard_set; + struct sc_keyboard_inject *ki = DOWNCAST(kp); if (event->repeat) { diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 5790310b79..1f8132f26c 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -18,8 +18,17 @@ struct sc_key_processor { }; struct sc_key_processor_ops { + + /** + * Process the keyboard event + * + * The flag `device_clipboard_set` indicates that the input manager sent a + * control message to synchronize the device clipboard as a result of this + * key event. + */ void - (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event); + (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, + bool device_clipboard_set); void (*process_text)(struct sc_key_processor *kp, From aba1fc03c3702404b30b7a8aebd2489f8b7fed1c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 14:33:13 +0100 Subject: [PATCH 4/9] Add acksync helper to wait for acks This will allow to send requests with sequence numbers to the server and wait for acknowledgements. PR #2814 --- app/meson.build | 1 + app/src/util/acksync.c | 76 ++++++++++++++++++++++++++++++++++++++++++ app/src/util/acksync.h | 61 +++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 app/src/util/acksync.c create mode 100644 app/src/util/acksync.h diff --git a/app/meson.build b/app/meson.build index 3a5cb12ade..d6fac58048 100644 --- a/app/meson.build +++ b/app/meson.build @@ -25,6 +25,7 @@ src = [ 'src/server.c', 'src/stream.c', 'src/video_buffer.c', + 'src/util/acksync.c', 'src/util/file.c', 'src/util/intr.c', 'src/util/log.c', diff --git a/app/src/util/acksync.c b/app/src/util/acksync.c new file mode 100644 index 0000000000..2899cdcbe0 --- /dev/null +++ b/app/src/util/acksync.c @@ -0,0 +1,76 @@ +#include "acksync.h" + +#include +#include "util/log.h" + +bool +sc_acksync_init(struct sc_acksync *as) { + bool ok = sc_mutex_init(&as->mutex); + if (!ok) { + return false; + } + + ok = sc_cond_init(&as->cond); + if (!ok) { + sc_mutex_destroy(&as->mutex); + return false; + } + + as->stopped = false; + as->ack = SC_SEQUENCE_INVALID; + + return true; +} + +void +sc_acksync_destroy(struct sc_acksync *as) { + sc_cond_destroy(&as->cond); + sc_mutex_destroy(&as->mutex); +} + +void +sc_acksync_ack(struct sc_acksync *as, uint64_t sequence) { + sc_mutex_lock(&as->mutex); + + // Acknowledgements must be monotonic + assert(sequence >= as->ack); + + as->ack = sequence; + sc_cond_signal(&as->cond); + + sc_mutex_unlock(&as->mutex); +} + +enum sc_acksync_wait_result +sc_acksync_wait(struct sc_acksync *as, uint64_t ack, sc_tick deadline) { + sc_mutex_lock(&as->mutex); + + bool timed_out = false; + while (!as->stopped && as->ack < ack && !timed_out) { + timed_out = !sc_cond_timedwait(&as->cond, &as->mutex, deadline); + } + + enum sc_acksync_wait_result ret; + if (as->stopped) { + ret = SC_ACKSYNC_WAIT_INTR; + } else if (as->ack >= ack) { + ret = SC_ACKSYNC_WAIT_OK; + } else { + assert(timed_out); + ret = SC_ACKSYNC_WAIT_TIMEOUT; + } + sc_mutex_unlock(&as->mutex); + + return ret; +} + +/** + * Interrupt any `sc_acksync_wait()` + */ +void +sc_acksync_interrupt(struct sc_acksync *as) { + sc_mutex_lock(&as->mutex); + as->stopped = true; + sc_cond_signal(&as->cond); + sc_mutex_unlock(&as->mutex); +} diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h new file mode 100644 index 0000000000..1fd3444496 --- /dev/null +++ b/app/src/util/acksync.h @@ -0,0 +1,61 @@ +#ifndef SC_ACK_SYNC_H +#define SC_ACK_SYNC_H + +#include "common.h" + +#include "thread.h" + +#define SC_SEQUENCE_INVALID 0 + +/** + * Helper to wait for acknowledgments + */ +struct sc_acksync { + sc_mutex mutex; + sc_cond cond; + + bool stopped; + + // Last acked value, initially SC_SEQUENCE_INVALID + uint64_t ack; +}; + +enum sc_acksync_wait_result { + // Acknowledgment received + SC_ACKSYNC_WAIT_OK, + + // Timeout expired + SC_ACKSYNC_WAIT_TIMEOUT, + + // Interrupted from another thread by sc_acksync_interrupt() + SC_ACKSYNC_WAIT_INTR, +}; + +bool +sc_acksync_init(struct sc_acksync *as); + +void +sc_acksync_destroy(struct sc_acksync *as); + +/** + * Acknowledge `sequence` + * + * The `sequence` must be greater than (or equal to) any previous acknowledged + * sequence. + */ +void +sc_acksync_ack(struct sc_acksync *as, uint64_t sequence); + +/** + * Wait for acknowledgment of sequence `ack` (or higher) + */ +enum sc_acksync_wait_result +sc_acksync_wait(struct sc_acksync *as, uint64_t ack, sc_tick deadline); + +/** + * Interrupt any `sc_acksync_wait()` + */ +void +sc_acksync_interrupt(struct sc_acksync *as); + +#endif From 901d8371655582b432d5d92430177a59df8058b9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 11:50:33 +0100 Subject: [PATCH 5/9] Add sequence number to set_clipboard request This will allow the client to request an acknowledgement. PR #2814 --- app/src/control_msg.c | 10 ++++++---- app/src/control_msg.h | 1 + app/src/input_manager.c | 1 + app/tests/test_control_msg_serialize.c | 4 +++- .../java/com/genymobile/scrcpy/ControlMessage.java | 8 +++++++- .../com/genymobile/scrcpy/ControlMessageReader.java | 7 ++++--- .../genymobile/scrcpy/ControlMessageReaderTest.java | 4 ++++ 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/app/src/control_msg.c b/app/src/control_msg.c index 74e3315c52..83ab0b7be1 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -118,11 +118,12 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) { buf[1] = msg->inject_keycode.action; return 2; case CONTROL_MSG_TYPE_SET_CLIPBOARD: { - buf[1] = !!msg->set_clipboard.paste; + buffer_write64be(&buf[1], msg->set_clipboard.sequence); + buf[9] = !!msg->set_clipboard.paste; size_t len = write_string(msg->set_clipboard.text, CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH, - &buf[2]); - return 2 + len; + &buf[10]); + return 10 + len; } case CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE: buf[1] = msg->set_screen_power_mode.mode; @@ -199,7 +200,8 @@ control_msg_log(const struct control_msg *msg) { KEYEVENT_ACTION_LABEL(msg->inject_keycode.action)); break; case CONTROL_MSG_TYPE_SET_CLIPBOARD: - LOG_CMSG("clipboard %s \"%s\"", + LOG_CMSG("clipboard %" PRIu64_ " %s \"%s\"", + msg->set_clipboard.sequence, msg->set_clipboard.paste ? "paste" : "copy", msg->set_clipboard.text); break; diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 16492849ab..7352defe7b 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -70,6 +70,7 @@ struct control_msg { // screen may only be turned on on ACTION_DOWN } back_or_screen_on; struct { + uint64_t sequence; char *text; // owned, to be freed by free() bool paste; } set_clipboard; diff --git a/app/src/input_manager.c b/app/src/input_manager.c index 16c822344b..e7afee77da 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -225,6 +225,7 @@ set_device_clipboard(struct controller *controller, bool paste) { struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD; + msg.set_clipboard.sequence = 0; // unused for now msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index b237e28e4d..5cd7056b22 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -226,6 +226,7 @@ static void test_serialize_set_clipboard(void) { struct control_msg msg = { .type = CONTROL_MSG_TYPE_SET_CLIPBOARD, .set_clipboard = { + .sequence = UINT64_C(0x0102030405060708), .paste = true, .text = "hello, world!", }, @@ -233,10 +234,11 @@ static void test_serialize_set_clipboard(void) { unsigned char buf[CONTROL_MSG_MAX_SIZE]; size_t size = control_msg_serialize(&msg, buf); - assert(size == 19); + assert(size == 27); const unsigned char expected[] = { CONTROL_MSG_TYPE_SET_CLIPBOARD, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence 1, // paste 0x00, 0x00, 0x00, 0x0d, // text length 'h', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // text diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index f8edd53c11..a536ed91ac 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -31,6 +31,7 @@ public final class ControlMessage { private int vScroll; private boolean paste; private int repeat; + private long sequence; private ControlMessage() { } @@ -79,9 +80,10 @@ public static ControlMessage createBackOrScreenOn(int action) { return msg; } - public static ControlMessage createSetClipboard(String text, boolean paste) { + public static ControlMessage createSetClipboard(long sequence, String text, boolean paste) { ControlMessage msg = new ControlMessage(); msg.type = TYPE_SET_CLIPBOARD; + msg.sequence = sequence; msg.text = text; msg.paste = paste; return msg; @@ -154,4 +156,8 @@ public boolean getPaste() { public int getRepeat() { return repeat; } + + public long getSequence() { + return sequence; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index e4ab840201..80931e9426 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -13,11 +13,11 @@ public class ControlMessageReader { static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20; static final int BACK_OR_SCREEN_ON_LENGTH = 1; static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1; - static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 1; + static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 9; private static final int MESSAGE_MAX_SIZE = 1 << 18; // 256k - public static final int CLIPBOARD_TEXT_MAX_LENGTH = MESSAGE_MAX_SIZE - 6; // type: 1 byte; paste flag: 1 byte; length: 4 bytes + public static final int CLIPBOARD_TEXT_MAX_LENGTH = MESSAGE_MAX_SIZE - 14; // type: 1 byte; sequence: 8 bytes; paste flag: 1 byte; length: 4 bytes public static final int INJECT_TEXT_MAX_LENGTH = 300; private final byte[] rawBuffer = new byte[MESSAGE_MAX_SIZE]; @@ -166,12 +166,13 @@ private ControlMessage parseSetClipboard() { if (buffer.remaining() < SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH) { return null; } + long sequence = buffer.getLong(); boolean paste = buffer.get() != 0; String text = parseString(); if (text == null) { return null; } - return ControlMessage.createSetClipboard(text, paste); + return ControlMessage.createSetClipboard(sequence, text, paste); } private ControlMessage parseSetScreenPowerMode() { diff --git a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java index 7f3d3f61fa..3b0b6c0da8 100644 --- a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java @@ -235,6 +235,7 @@ public void testParseSetClipboardEvent() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); dos.writeByte(ControlMessage.TYPE_SET_CLIPBOARD); + dos.writeLong(0x0102030405060708L); // sequence dos.writeByte(1); // paste byte[] text = "testé".getBytes(StandardCharsets.UTF_8); dos.writeInt(text.length); @@ -246,6 +247,7 @@ public void testParseSetClipboardEvent() throws IOException { ControlMessage event = reader.next(); Assert.assertEquals(ControlMessage.TYPE_SET_CLIPBOARD, event.getType()); + Assert.assertEquals(0x0102030405060708L, event.getSequence()); Assert.assertEquals("testé", event.getText()); Assert.assertTrue(event.getPaste()); } @@ -259,6 +261,7 @@ public void testParseBigSetClipboardEvent() throws IOException { dos.writeByte(ControlMessage.TYPE_SET_CLIPBOARD); byte[] rawText = new byte[ControlMessageReader.CLIPBOARD_TEXT_MAX_LENGTH]; + dos.writeLong(0x0807060504030201L); // sequence dos.writeByte(1); // paste Arrays.fill(rawText, (byte) 'a'); String text = new String(rawText, 0, rawText.length); @@ -272,6 +275,7 @@ public void testParseBigSetClipboardEvent() throws IOException { ControlMessage event = reader.next(); Assert.assertEquals(ControlMessage.TYPE_SET_CLIPBOARD, event.getType()); + Assert.assertEquals(0x0807060504030201L, event.getSequence()); Assert.assertEquals(text, event.getText()); Assert.assertTrue(event.getPaste()); } From 2a0730ee9bacc8df92c7c38ce83a735182d685e0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 12:10:09 +0100 Subject: [PATCH 6/9] Add device clipboard set acknowledgement Add a device message type so that the device could send acknowledgements for SET_CLIPBOARD requests. PR #2814 --- app/src/device_msg.c | 6 ++++++ app/src/device_msg.h | 4 ++++ app/src/receiver.c | 3 +++ app/tests/test_device_msg_deserialize.c | 15 ++++++++++++++ .../com/genymobile/scrcpy/DeviceMessage.java | 13 ++++++++++++ .../scrcpy/DeviceMessageWriter.java | 6 +++++- .../scrcpy/DeviceMessageWriterTest.java | 20 +++++++++++++++++++ 7 files changed, 66 insertions(+), 1 deletion(-) diff --git a/app/src/device_msg.c b/app/src/device_msg.c index 827f421379..4163b9fc9f 100644 --- a/app/src/device_msg.c +++ b/app/src/device_msg.c @@ -1,5 +1,6 @@ #include "device_msg.h" +#include #include #include @@ -34,6 +35,11 @@ device_msg_deserialize(const unsigned char *buf, size_t len, msg->clipboard.text = text; return 5 + clipboard_len; } + case DEVICE_MSG_TYPE_ACK_CLIPBOARD: { + uint64_t sequence = buffer_read64be(&buf[1]); + msg->ack_clipboard.sequence = sequence; + return 9; + } default: LOGW("Unknown device message type: %d", (int) msg->type); return -1; // error, we cannot recover diff --git a/app/src/device_msg.h b/app/src/device_msg.h index 888d9216ad..a0c989e3f8 100644 --- a/app/src/device_msg.h +++ b/app/src/device_msg.h @@ -13,6 +13,7 @@ enum device_msg_type { DEVICE_MSG_TYPE_CLIPBOARD, + DEVICE_MSG_TYPE_ACK_CLIPBOARD, }; struct device_msg { @@ -21,6 +22,9 @@ struct device_msg { struct { char *text; // owned, to be freed by free() } clipboard; + struct { + uint64_t sequence; + } ack_clipboard; }; }; diff --git a/app/src/receiver.c b/app/src/receiver.c index b5cf9b395b..52e6034d37 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -37,6 +37,9 @@ process_msg(struct device_msg *msg) { SDL_SetClipboardText(msg->clipboard.text); break; } + case DEVICE_MSG_TYPE_ACK_CLIPBOARD: + // TODO + break; } } diff --git a/app/tests/test_device_msg_deserialize.c b/app/tests/test_device_msg_deserialize.c index 3427d64045..835096c000 100644 --- a/app/tests/test_device_msg_deserialize.c +++ b/app/tests/test_device_msg_deserialize.c @@ -47,11 +47,26 @@ static void test_deserialize_clipboard_big(void) { device_msg_destroy(&msg); } +static void test_deserialize_ack_set_clipboard(void) { + const unsigned char input[] = { + DEVICE_MSG_TYPE_ACK_CLIPBOARD, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // sequence + }; + + struct device_msg msg; + ssize_t r = device_msg_deserialize(input, sizeof(input), &msg); + assert(r == 9); + + assert(msg.type == DEVICE_MSG_TYPE_ACK_CLIPBOARD); + assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708)); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; test_deserialize_clipboard(); test_deserialize_clipboard_big(); + test_deserialize_ack_set_clipboard(); return 0; } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java index c6eebd3806..2e333e3fe0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java @@ -3,9 +3,11 @@ public final class DeviceMessage { public static final int TYPE_CLIPBOARD = 0; + public static final int TYPE_ACK_CLIPBOARD = 1; private int type; private String text; + private long sequence; private DeviceMessage() { } @@ -17,6 +19,13 @@ public static DeviceMessage createClipboard(String text) { return event; } + public static DeviceMessage createAckClipboard(long sequence) { + DeviceMessage event = new DeviceMessage(); + event.type = TYPE_ACK_CLIPBOARD; + event.sequence = sequence; + return event; + } + public int getType() { return type; } @@ -24,4 +33,8 @@ public int getType() { public String getText() { return text; } + + public long getSequence() { + return sequence; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java index 15d91a35b7..bcd8d20676 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java @@ -15,7 +15,7 @@ public class DeviceMessageWriter { public void writeTo(DeviceMessage msg, OutputStream output) throws IOException { buffer.clear(); - buffer.put((byte) DeviceMessage.TYPE_CLIPBOARD); + buffer.put((byte) msg.getType()); switch (msg.getType()) { case DeviceMessage.TYPE_CLIPBOARD: String text = msg.getText(); @@ -25,6 +25,10 @@ public void writeTo(DeviceMessage msg, OutputStream output) throws IOException { buffer.put(raw, 0, len); output.write(rawBuffer, 0, buffer.position()); break; + case DeviceMessage.TYPE_ACK_CLIPBOARD: + buffer.putLong(msg.getSequence()); + output.write(rawBuffer, 0, buffer.position()); + break; default: Ln.w("Unknown device message: " + msg.getType()); break; diff --git a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java index 88bf2af9fa..7b917d337c 100644 --- a/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java @@ -32,4 +32,24 @@ public void testSerializeClipboard() throws IOException { Assert.assertArrayEquals(expected, actual); } + + @Test + public void testSerializeAckSetClipboard() throws IOException { + DeviceMessageWriter writer = new DeviceMessageWriter(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(DeviceMessage.TYPE_ACK_CLIPBOARD); + dos.writeLong(0x0102030405060708L); + + byte[] expected = bos.toByteArray(); + + DeviceMessage msg = DeviceMessage.createAckClipboard(0x0102030405060708L); + bos = new ByteArrayOutputStream(); + writer.writeTo(msg, bos); + + byte[] actual = bos.toByteArray(); + + Assert.assertArrayEquals(expected, actual); + } } From 41abe021e2a73efd4899b0efcd0b9eef9ec68c9b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 20 Nov 2021 12:23:45 +0100 Subject: [PATCH 7/9] Make the device acknowledge device clipboard If the client provided a sequence number on SET_CLIPBOARD request, make the device send back an acknowledgement once the clipboard is set. PR #2814 --- .../com/genymobile/scrcpy/ControlMessage.java | 2 ++ .../com/genymobile/scrcpy/Controller.java | 5 ++++ .../com/genymobile/scrcpy/DeviceMessage.java | 2 ++ .../scrcpy/DeviceMessageSender.java | 24 ++++++++++++++++--- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index a536ed91ac..2cd8019197 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -18,6 +18,8 @@ public final class ControlMessage { public static final int TYPE_SET_SCREEN_POWER_MODE = 10; public static final int TYPE_ROTATE_DEVICE = 11; + public static final long SEQUENCE_INVALID = 0; + private int type; private String text; private int metaState; // KeyEvent.META_* diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 45882bb9c9..8b24b300cd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -120,7 +120,12 @@ private void handleEvent() throws IOException { } break; case ControlMessage.TYPE_SET_CLIPBOARD: + long sequence = msg.getSequence(); setClipboard(msg.getText(), msg.getPaste()); + if (sequence != ControlMessage.SEQUENCE_INVALID) { + // Acknowledgement requested + sender.pushAckClipboard(sequence); + } break; case ControlMessage.TYPE_SET_SCREEN_POWER_MODE: if (device.supportsInputEvents()) { diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java index 2e333e3fe0..5b7c4de5b5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java @@ -5,6 +5,8 @@ public final class DeviceMessage { public static final int TYPE_CLIPBOARD = 0; public static final int TYPE_ACK_CLIPBOARD = 1; + public static final long SEQUENCE_INVALID = ControlMessage.SEQUENCE_INVALID; + private int type; private String text; private long sequence; diff --git a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java index bbf4dd2eef..4ebccaccf6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java +++ b/server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java @@ -8,6 +8,8 @@ public final class DeviceMessageSender { private String clipboardText; + private long ack; + public DeviceMessageSender(DesktopConnection connection) { this.connection = connection; } @@ -17,18 +19,34 @@ public synchronized void pushClipboardText(String text) { notify(); } + public synchronized void pushAckClipboard(long sequence) { + ack = sequence; + notify(); + } + public void loop() throws IOException, InterruptedException { while (true) { String text; + long sequence; synchronized (this) { - while (clipboardText == null) { + while (ack == DeviceMessage.SEQUENCE_INVALID && clipboardText == null) { wait(); } text = clipboardText; clipboardText = null; + + sequence = ack; + ack = DeviceMessage.SEQUENCE_INVALID; + } + + if (sequence != DeviceMessage.SEQUENCE_INVALID) { + DeviceMessage event = DeviceMessage.createAckClipboard(sequence); + connection.sendDeviceMessage(event); + } + if (text != null) { + DeviceMessage event = DeviceMessage.createClipboard(text); + connection.sendDeviceMessage(event); } - DeviceMessage event = DeviceMessage.createClipboard(text); - connection.sendDeviceMessage(event); } } } From 2d5525eac128c6bb02344d733ab487b3aedd2431 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 21 Nov 2021 22:11:08 +0100 Subject: [PATCH 8/9] Move PRIu64 Windows workaround to compat.h So that we can use it from several files. PR #2814 --- app/src/compat.h | 6 ++++++ app/src/control_msg.c | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/src/compat.h b/app/src/compat.h index 32759c01b4..c06c23fae2 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -6,6 +6,12 @@ #include #include +#ifndef __WIN32 +# define PRIu64_ PRIu64 +#else +# define PRIu64_ "I64u" // Windows... +#endif + // In ffmpeg/doc/APIchanges: // 2018-02-06 - 0694d87024 - lavf 58.9.100 - avformat.h // Deprecate use of av_register_input_format(), av_register_output_format(), diff --git a/app/src/control_msg.c b/app/src/control_msg.c index 83ab0b7be1..90cde0cfa2 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -171,11 +171,6 @@ control_msg_log(const struct control_msg *msg) { (long) msg->inject_touch_event.buttons); } else { // numeric pointer id -#ifndef __WIN32 -# define PRIu64_ PRIu64 -#else -# define PRIu64_ "I64u" // Windows... -#endif LOG_CMSG("touch [id=%" PRIu64_ "] %-4s position=%" PRIi32 ",%" PRIi32 " pressure=%g buttons=%06lx", id, From 5d17bcf1bc38690739c88ad2b0a069a67c8bedce Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 21 Nov 2021 17:40:11 +0100 Subject: [PATCH 9/9] Wait SET_CLIPBOARD ack before Ctrl+v via HID To allow seamless copy-paste, on Ctrl+v, a SET_CLIPBOARD request is performed before injecting Ctrl+v. But when HID keyboard is enabled, the Ctrl+v injection is not sent on the same channel as the clipboard request, so they are not serialized, and may occur in any order. If Ctrl+v happens to be injected before the new clipboard content is set, then the old content is pasted instead, which is incorrect. To minimize the probability of occurrence of the wrong order, a delay of 2 milliseconds was added before injecting Ctrl+v. Then 5ms. But even with 5ms, the wrong behavior sometimes happens. To handle it properly, add an acknowledgement mechanism, so that Ctrl+v is injected over AOA only after the SET_CLIPBOARD request has been performed and acknowledged by the server. Refs e4163321f00bb3830c6049bdb6c1515e7cc668a0 Refs 45b0f8123a52f5c73a5860d616f4ceba2766ca6a PR #2814 --- app/src/aoa_hid.c | 41 ++++++++++++++++++++++------------- app/src/aoa_hid.h | 7 ++++-- app/src/controller.c | 5 +++-- app/src/controller.h | 4 +++- app/src/hid_keyboard.c | 16 +++++++++----- app/src/input_manager.c | 30 ++++++++++++++++++++----- app/src/input_manager.h | 2 ++ app/src/keyboard_inject.c | 6 +++-- app/src/receiver.c | 19 +++++++++++----- app/src/receiver.h | 6 ++++- app/src/scrcpy.c | 23 ++++++++++++++++++-- app/src/trait/key_processor.h | 18 +++++++++++---- app/src/util/acksync.h | 5 +++++ 13 files changed, 135 insertions(+), 47 deletions(-) diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c index 4c0b2bdaa0..6fd32610cb 100644 --- a/app/src/aoa_hid.c +++ b/app/src/aoa_hid.c @@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, hid_event->accessory_id = accessory_id; hid_event->buffer = buffer; hid_event->size = buffer_size; - hid_event->delay = 0; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; } void @@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { } bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial) { +sc_aoa_init(struct sc_aoa *aoa, const char *serial, + struct sc_acksync *acksync) { + assert(acksync); + cbuf_init(&aoa->queue); if (!sc_mutex_init(&aoa->mutex)) { @@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) { } aoa->stopped = false; + aoa->acksync = acksync; return true; } @@ -332,23 +336,28 @@ run_aoa_thread(void *data) { assert(non_empty); (void) non_empty; - assert(event.delay >= 0); - if (event.delay) { - // Wait during the specified delay before injecting the HID event - sc_tick deadline = sc_tick_now() + event.delay; - bool timed_out = false; - while (!aoa->stopped && !timed_out) { - timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex, - deadline); - } - if (aoa->stopped) { - sc_mutex_unlock(&aoa->mutex); + uint64_t ack_to_wait = event.ack_to_wait; + sc_mutex_unlock(&aoa->mutex); + + if (ack_to_wait != SC_SEQUENCE_INVALID) { + LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait); + // Do not block the loop indefinitely if the ack never comes (it should + // never happen) + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500); + enum sc_acksync_wait_result result = + sc_acksync_wait(aoa->acksync, ack_to_wait, deadline); + + if (result == SC_ACKSYNC_WAIT_TIMEOUT) { + LOGW("Ack not received after 500ms, discarding HID event"); + sc_hid_event_destroy(&event); + continue; + } else if (result == SC_ACKSYNC_WAIT_INTR) { + // stopped + sc_hid_event_destroy(&event); break; } } - sc_mutex_unlock(&aoa->mutex); - bool ok = sc_aoa_send_hid_event(aoa, &event); sc_hid_event_destroy(&event); if (!ok) { @@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) { aoa->stopped = true; sc_cond_signal(&aoa->event_cond); sc_mutex_unlock(&aoa->mutex); + + sc_acksync_interrupt(aoa->acksync); } void diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h index 24cef50212..e8fb970822 100644 --- a/app/src/aoa_hid.h +++ b/app/src/aoa_hid.h @@ -6,6 +6,7 @@ #include +#include "util/acksync.h" #include "util/cbuf.h" #include "util/thread.h" #include "util/tick.h" @@ -14,7 +15,7 @@ struct sc_hid_event { uint16_t accessory_id; unsigned char *buffer; uint16_t size; - sc_tick delay; + uint64_t ack_to_wait; }; // Takes ownership of buffer @@ -36,10 +37,12 @@ struct sc_aoa { sc_cond event_cond; bool stopped; struct sc_hid_event_queue queue; + + struct sc_acksync *acksync; }; bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial); +sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync); void sc_aoa_destroy(struct sc_aoa *aoa); diff --git a/app/src/controller.c b/app/src/controller.c index e486ea7224..6cf3d20e47 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -5,10 +5,11 @@ #include "util/log.h" bool -controller_init(struct controller *controller, sc_socket control_socket) { +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync) { cbuf_init(&controller->queue); - bool ok = receiver_init(&controller->receiver, control_socket); + bool ok = receiver_init(&controller->receiver, control_socket, acksync); if (!ok) { return false; } diff --git a/app/src/controller.h b/app/src/controller.h index e700413100..00267878ac 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -7,6 +7,7 @@ #include "control_msg.h" #include "receiver.h" +#include "util/acksync.h" #include "util/cbuf.h" #include "util/net.h" #include "util/thread.h" @@ -24,7 +25,8 @@ struct controller { }; bool -controller_init(struct controller *controller, sc_socket control_socket); +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync); void controller_destroy(struct controller *controller); diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c index 2809ccd6bf..2afc09b2f4 100644 --- a/app/src/hid_keyboard.c +++ b/app/src/hid_keyboard.c @@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) { static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { if (event->repeat) { // In USB HID protocol, key repeat is handled by the host (Android), so // just ignore key repeat here. @@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - if (device_clipboard_set) { + if (ack_to_wait) { // Ctrl+v is pressed, so clipboard synchronization has been - // requested. Wait a bit so that the clipboard is set before - // injecting Ctrl+v via HID, otherwise it would paste the old - // clipboard content. - hid_event.delay = SC_TICK_FROM_MS(5); + // requested. Wait until clipboard synchronization is acknowledged + // by the server, otherwise it could paste the old clipboard + // content. + hid_event.ack_to_wait = ack_to_wait; } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { @@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { .process_text = sc_key_processor_process_text, }; + // Clipboard synchronization is requested over the control socket, while HID + // events are sent over AOA, so it must wait for clipboard synchronization + // to be acknowledged by the device before injecting Ctrl+v. + kb->key_processor.async_paste = true; kb->key_processor.ops = &ops; return true; diff --git a/app/src/input_manager.c b/app/src/input_manager.c index e7afee77da..f1715b79f0 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller, im->last_keycode = SDLK_UNKNOWN; im->last_mod = 0; im->key_repeat = 0; + + im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID } static void @@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) { } static bool -set_device_clipboard(struct controller *controller, bool paste) { +set_device_clipboard(struct controller *controller, bool paste, + uint64_t sequence) { char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); @@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) { struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD; - msg.set_clipboard.sequence = 0; // unused for now + msg.set_clipboard.sequence = sequence; msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; @@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im, // inject the text as input events clipboard_paste(controller); } else { - // store the text in the device clipboard and paste - set_device_clipboard(controller, true); + // store the text in the device clipboard and paste, + // without requesting an acknowledgment + set_device_clipboard(controller, true, + SC_SEQUENCE_INVALID); } } return; @@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im, return; } + uint64_t ack_to_wait = SC_SEQUENCE_INVALID; bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat; if (is_ctrl_v) { if (im->legacy_paste) { @@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im, clipboard_paste(controller); return; } + + // Request an acknowledgement only if necessary + uint64_t sequence = im->kp->async_paste ? im->next_sequence + : SC_SEQUENCE_INVALID; + // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - bool ok = set_device_clipboard(controller, false); + bool ok = set_device_clipboard(controller, false, sequence); if (!ok) { LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); return; } + + if (im->kp->async_paste) { + // The key processor must wait for this ack before injecting Ctrl+v + ack_to_wait = sequence; + // Increment only when the request succeeded + ++im->next_sequence; + } } - im->kp->ops->process_key(im->kp, event, is_ctrl_v); + im->kp->ops->process_key(im->kp, event, ack_to_wait); } static void diff --git a/app/src/input_manager.h b/app/src/input_manager.h index f018f98a85..22d773811c 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -38,6 +38,8 @@ struct input_manager { unsigned key_repeat; SDL_Keycode last_keycode; uint16_t last_mod; + + uint64_t next_sequence; // used for request acknowledgements }; void diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c index e59b5520e9..4112fd4cb7 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_inject.c @@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { // The device clipboard synchronization and the key event messages are // serialized, there is nothing special to do to ensure that the clipboard // is set before injecting Ctrl+v. - (void) device_clipboard_set; + (void) ack_to_wait; struct sc_keyboard_inject *ki = DOWNCAST(kp); @@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki, .process_text = sc_key_processor_process_text, }; + // Key injection and clipboard synchronization are serialized + ki->key_processor.async_paste = false; ki->key_processor.ops = &ops; } diff --git a/app/src/receiver.c b/app/src/receiver.c index 52e6034d37..eeb206f173 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -7,12 +7,16 @@ #include "util/log.h" bool -receiver_init(struct receiver *receiver, sc_socket control_socket) { +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync) { bool ok = sc_mutex_init(&receiver->mutex); if (!ok) { return false; } + receiver->control_socket = control_socket; + receiver->acksync = acksync; + return true; } @@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) { } static void -process_msg(struct device_msg *msg) { +process_msg(struct receiver *receiver, struct device_msg *msg) { switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { char *current = SDL_GetClipboardText(); @@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) { break; } case DEVICE_MSG_TYPE_ACK_CLIPBOARD: - // TODO + assert(receiver->acksync); + LOGD("Ack device clipboard sequence=%" PRIu64_, + msg->ack_clipboard.sequence); + sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); break; } } static ssize_t -process_msgs(const unsigned char *buf, size_t len) { +process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) { size_t head = 0; for (;;) { struct device_msg msg; @@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) { return head; } - process_msg(&msg); + process_msg(receiver, &msg); device_msg_destroy(&msg); head += r; @@ -84,7 +91,7 @@ run_receiver(void *data) { } head += r; - ssize_t consumed = process_msgs(buf, head); + ssize_t consumed = process_msgs(receiver, buf, head); if (consumed == -1) { // an error occurred break; diff --git a/app/src/receiver.h b/app/src/receiver.h index 99f128a4be..3c4e8c645a 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -5,6 +5,7 @@ #include +#include "util/acksync.h" #include "util/net.h" #include "util/thread.h" @@ -14,10 +15,13 @@ struct receiver { sc_socket control_socket; sc_thread thread; sc_mutex mutex; + + struct sc_acksync *acksync; }; bool -receiver_init(struct receiver *receiver, sc_socket control_socket); +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync); void receiver_destroy(struct receiver *receiver); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 61087681c6..04239bf540 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,7 @@ #include "screen.h" #include "server.h" #include "stream.h" +#include "util/acksync.h" #include "util/log.h" #include "util/net.h" #ifdef HAVE_V4L2 @@ -46,6 +47,8 @@ struct scrcpy { struct file_handler file_handler; #ifdef HAVE_AOA_HID struct sc_aoa aoa; + // sequence/ack helper to synchronize clipboard and Ctrl+v via HID + struct sc_acksync acksync; #endif union { struct sc_keyboard_inject keyboard_inject; @@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) { bool controller_started = false; bool screen_initialized = false; + struct sc_acksync *acksync = NULL; + struct sc_server_params params = { .serial = options->serial, .log_level = options->log_level, @@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) { } if (options->control) { - if (!controller_init(&s->controller, s->server.control_socket)) { +#ifdef HAVE_AOA_HID + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) { + bool ok = sc_acksync_init(&s->acksync); + if (!ok) { + goto end; + } + + acksync = &s->acksync; + } +#endif + if (!controller_init(&s->controller, s->server.control_socket, + acksync)) { goto end; } controller_initialized = true; @@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool aoa_hid_ok = false; - bool ok = sc_aoa_init(&s->aoa, serial); + bool ok = sc_aoa_init(&s->aoa, serial, acksync); if (!ok) { goto aoa_hid_end; } @@ -586,6 +602,9 @@ scrcpy(struct scrcpy_options *options) { sc_hid_keyboard_destroy(&s->keyboard_hid); sc_aoa_stop(&s->aoa); } + if (acksync) { + sc_acksync_destroy(acksync); + } #endif if (controller_started) { controller_stop(&s->controller); diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 1f8132f26c..f4afe27b17 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -14,6 +14,15 @@ * Component able to process and inject keys should implement this trait. */ struct sc_key_processor { + /** + * Set by the implementation to indicate that it must explicitly wait for + * the clipboard to be set on the device before injecting Ctrl+v to avoid + * race conditions. If it is set, the input_manager will pass a valid + * ack_to_wait to process_key() in case of clipboard synchronization + * resulting of the key event. + */ + bool async_paste; + const struct sc_key_processor_ops *ops; }; @@ -22,13 +31,14 @@ struct sc_key_processor_ops { /** * Process the keyboard event * - * The flag `device_clipboard_set` indicates that the input manager sent a - * control message to synchronize the device clipboard as a result of this - * key event. + * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates + * the acknowledgement number to wait for before injecting this event. + * This allows to ensure that the device clipboard is set before injecting + * Ctrl+v on the device. */ void (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set); + uint64_t ack_to_wait); void (*process_text)(struct sc_key_processor *kp, diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h index 1fd3444496..58ab1b353c 100644 --- a/app/src/util/acksync.h +++ b/app/src/util/acksync.h @@ -9,6 +9,11 @@ /** * Helper to wait for acknowledgments + * + * In practice, it is used to wait for device clipboard acknowledgement from the + * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the + * content of the old device clipboard (if Ctrl+v was injected before the + * clipboard content was actually set). */ struct sc_acksync { sc_mutex mutex;