Wait SET_CLIPBOARD ack before Ctrl+v via HID

To allow seamless copy-paste, on Ctrl+v, a SET_CLIPBOARD request is
performed before injecting Ctrl+v.

But when HID keyboard is enabled, the Ctrl+v injection is not sent on
the same channel as the clipboard request, so they are not serialized,
and may occur in any order. If Ctrl+v happens to be injected before the
new clipboard content is set, then the old content is pasted instead,
which is incorrect.

To minimize the probability of occurrence of the wrong order, a delay of
2 milliseconds was added before injecting Ctrl+v. Then 5ms. But even
with 5ms, the wrong behavior sometimes happens.

To handle it properly, add an acknowledgement mechanism, so that Ctrl+v
is injected over AOA only after the SET_CLIPBOARD request has been
performed and acknowledged by the server.

Refs e416332
Refs 45b0f81

PR #2814 <#2814>

app/src/aoa_hid.c

@@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
     hid_event->accessory_id = accessory_id;
     hid_event->buffer = buffer;
     hid_event->size = buffer_size;
-    hid_event->delay = 0;
+    hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
 }
 
 void
@@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
 }
 
 bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
+sc_aoa_init(struct sc_aoa *aoa, const char *serial,
+            struct sc_acksync *acksync) {
+    assert(acksync);
+
     cbuf_init(&aoa->queue);
 
     if (!sc_mutex_init(&aoa->mutex)) {
@@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
     }
 
     aoa->stopped = false;
+    aoa->acksync = acksync;
 
     return true;
 }
@@ -332,23 +336,28 @@ run_aoa_thread(void *data) {
         assert(non_empty);
         (void) non_empty;
 
-        assert(event.delay >= 0);
-        if (event.delay) {
-            // Wait during the specified delay before injecting the HID event
-            sc_tick deadline = sc_tick_now() + event.delay;
-            bool timed_out = false;
-            while (!aoa->stopped && !timed_out) {
-                timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex,
-                                               deadline);
-            }
-            if (aoa->stopped) {
-                sc_mutex_unlock(&aoa->mutex);
+        uint64_t ack_to_wait = event.ack_to_wait;
+        sc_mutex_unlock(&aoa->mutex);
+
+        if (ack_to_wait != SC_SEQUENCE_INVALID) {
+            LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait);
+            // Do not block the loop indefinitely if the ack never comes (it should
+            // never happen)
+            sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500);
+            enum sc_acksync_wait_result result =
+                sc_acksync_wait(aoa->acksync, ack_to_wait, deadline);
+
+            if (result == SC_ACKSYNC_WAIT_TIMEOUT) {
+                LOGW("Ack not received after 500ms, discarding HID event");
+                sc_hid_event_destroy(&event);
+                continue;
+            } else if (result == SC_ACKSYNC_WAIT_INTR) {
+                // stopped
+                sc_hid_event_destroy(&event);
                 break;
             }
         }
 
-        sc_mutex_unlock(&aoa->mutex);
-
         bool ok = sc_aoa_send_hid_event(aoa, &event);
         sc_hid_event_destroy(&event);
         if (!ok) {
@@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) {
     aoa->stopped = true;
     sc_cond_signal(&aoa->event_cond);
     sc_mutex_unlock(&aoa->mutex);
+
+    sc_acksync_interrupt(aoa->acksync);
 }
 
 void

app/src/aoa_hid.h

@@ -6,6 +6,7 @@
 
 #include <libusb-1.0/libusb.h>
 
+#include "util/acksync.h"
 #include "util/cbuf.h"
 #include "util/thread.h"
 #include "util/tick.h"
@@ -14,7 +15,7 @@ struct sc_hid_event {
     uint16_t accessory_id;
     unsigned char *buffer;
     uint16_t size;
-    sc_tick delay;
+    uint64_t ack_to_wait;
 };
 
 // Takes ownership of buffer
@@ -36,10 +37,12 @@ struct sc_aoa {
     sc_cond event_cond;
     bool stopped;
     struct sc_hid_event_queue queue;
+
+    struct sc_acksync *acksync;
 };
 
 bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial);
+sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync);
 
 void
 sc_aoa_destroy(struct sc_aoa *aoa);

app/src/controller.c

@@ -5,10 +5,11 @@
 #include "util/log.h"
 
 bool
-controller_init(struct controller *controller, sc_socket control_socket) {
+controller_init(struct controller *controller, sc_socket control_socket,
+                struct sc_acksync *acksync) {
     cbuf_init(&controller->queue);
 
-    bool ok = receiver_init(&controller->receiver, control_socket);
+    bool ok = receiver_init(&controller->receiver, control_socket, acksync);
     if (!ok) {
         return false;
     }

app/src/controller.h

@@ -7,6 +7,7 @@
 
 #include "control_msg.h"
 #include "receiver.h"
+#include "util/acksync.h"
 #include "util/cbuf.h"
 #include "util/net.h"
 #include "util/thread.h"
@@ -24,7 +25,8 @@ struct controller {
 };
 
 bool
-controller_init(struct controller *controller, sc_socket control_socket);
+controller_init(struct controller *controller, sc_socket control_socket,
+                struct sc_acksync *acksync);
 
 void
 controller_destroy(struct controller *controller);

app/src/hid_keyboard.c

@@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) {
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const SDL_KeyboardEvent *event,
-                             bool device_clipboard_set) {
+                             uint64_t ack_to_wait) {
     if (event->repeat) {
         // In USB HID protocol, key repeat is handled by the host (Android), so
         // just ignore key repeat here.
@@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
             }
         }
 
-        if (device_clipboard_set) {
+        if (ack_to_wait) {
             // Ctrl+v is pressed, so clipboard synchronization has been
-            // requested. Wait a bit so that the clipboard is set before
-            // injecting Ctrl+v via HID, otherwise it would paste the old
-            // clipboard content.
-            hid_event.delay = SC_TICK_FROM_MS(5);
+            // requested. Wait until clipboard synchronization is acknowledged
+            // by the server, otherwise it could paste the old clipboard
+            // content.
+            hid_event.ack_to_wait = ack_to_wait;
         }
 
         if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
@@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
         .process_text = sc_key_processor_process_text,
     };
 
+    // Clipboard synchronization is requested over the control socket, while HID
+    // events are sent over AOA, so it must wait for clipboard synchronization
+    // to be acknowledged by the device before injecting Ctrl+v.
+    kb->key_processor.async_paste = true;
     kb->key_processor.ops = &ops;
 
     return true;

app/src/input_manager.c

@@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller,
     im->last_keycode = SDLK_UNKNOWN;
     im->last_mod = 0;
     im->key_repeat = 0;
+
+    im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID
 }
 
 static void
@@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) {
 }
 
 static bool
-set_device_clipboard(struct controller *controller, bool paste) {
+set_device_clipboard(struct controller *controller, bool paste,
+                     uint64_t sequence) {
     char *text = SDL_GetClipboardText();
     if (!text) {
         LOGW("Could not get clipboard text: %s", SDL_GetError());
@@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) {
 
     struct control_msg msg;
     msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD;
-    msg.set_clipboard.sequence = 0; // unused for now
+    msg.set_clipboard.sequence = sequence;
     msg.set_clipboard.text = text_dup;
     msg.set_clipboard.paste = paste;
 
@@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im,
                         // inject the text as input events
                         clipboard_paste(controller);
                     } else {
-                        // store the text in the device clipboard and paste
-                        set_device_clipboard(controller, true);
+                        // store the text in the device clipboard and paste,
+                        // without requesting an acknowledgment
+                        set_device_clipboard(controller, true,
+                                             SC_SEQUENCE_INVALID);
                     }
                 }
                 return;
@@ -511,23 +516,36 @@ input_manager_process_key(struct input_manager *im,
         return;
     }
 
+    uint64_t ack_to_wait = SC_SEQUENCE_INVALID;
     bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat;
     if (is_ctrl_v) {
         if (im->legacy_paste) {
             // inject the text as input events
             clipboard_paste(controller);
             return;
         }
+
+        // Request an acknowledgement only if necessary
+        uint64_t sequence = im->kp->async_paste ? im->next_sequence
+                                                : SC_SEQUENCE_INVALID;
+
         // Synchronize the computer clipboard to the device clipboard before
         // sending Ctrl+v, to allow seamless copy-paste.
-        bool ok = set_device_clipboard(controller, false);
+        bool ok = set_device_clipboard(controller, false, sequence);
         if (!ok) {
             LOGW("Clipboard could not be synchronized, Ctrl+v not injected");
             return;
         }
+
+        if (im->kp->async_paste) {
+            // The key processor must wait for this ack before injecting Ctrl+v
+            ack_to_wait = sequence;
+            // Increment only when the request succeeded
+            ++im->next_sequence;
+        }
     }
 
-    im->kp->ops->process_key(im->kp, event, is_ctrl_v);
+    im->kp->ops->process_key(im->kp, event, ack_to_wait);
 }
 
 static void

app/src/input_manager.h

@@ -38,6 +38,8 @@ struct input_manager {
     unsigned key_repeat;
     SDL_Keycode last_keycode;
     uint16_t last_mod;
+
+    uint64_t next_sequence; // used for request acknowledgements
 };
 
 void

app/src/keyboard_inject.c

@@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to,
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const SDL_KeyboardEvent *event,
-                             bool device_clipboard_set) {
+                             uint64_t ack_to_wait) {
     // The device clipboard synchronization and the key event messages are
     // serialized, there is nothing special to do to ensure that the clipboard
     // is set before injecting Ctrl+v.
-    (void) device_clipboard_set;
+    (void) ack_to_wait;
 
     struct sc_keyboard_inject *ki = DOWNCAST(kp);
 
@@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki,
         .process_text = sc_key_processor_process_text,
     };
 
+    // Key injection and clipboard synchronization are serialized
+    ki->key_processor.async_paste = false;
     ki->key_processor.ops = &ops;
 }

app/src/receiver.c

@@ -7,12 +7,16 @@
 #include "util/log.h"
 
 bool
-receiver_init(struct receiver *receiver, sc_socket control_socket) {
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+              struct sc_acksync *acksync) {
     bool ok = sc_mutex_init(&receiver->mutex);
     if (!ok) {
         return false;
     }
+
     receiver->control_socket = control_socket;
+    receiver->acksync = acksync;
+
     return true;
 }
 
@@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) {
 }
 
 static void
-process_msg(struct device_msg *msg) {
+process_msg(struct receiver *receiver, struct device_msg *msg) {
     switch (msg->type) {
         case DEVICE_MSG_TYPE_CLIPBOARD: {
             char *current = SDL_GetClipboardText();
@@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) {
             break;
         }
         case DEVICE_MSG_TYPE_ACK_CLIPBOARD:
-            // TODO
+            assert(receiver->acksync);
+            LOGD("Ack device clipboard sequence=%" PRIu64_,
+                 msg->ack_clipboard.sequence);
+            sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
             break;
     }
 }
 
 static ssize_t
-process_msgs(const unsigned char *buf, size_t len) {
+process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) {
     size_t head = 0;
     for (;;) {
         struct device_msg msg;
@@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) {
             return head;
         }
 
-        process_msg(&msg);
+        process_msg(receiver, &msg);
         device_msg_destroy(&msg);
 
         head += r;
@@ -84,7 +91,7 @@ run_receiver(void *data) {
         }
 
         head += r;
-        ssize_t consumed = process_msgs(buf, head);
+        ssize_t consumed = process_msgs(receiver, buf, head);
         if (consumed == -1) {
             // an error occurred
             break;

app/src/receiver.h

@@ -5,6 +5,7 @@
 
 #include <stdbool.h>
 
+#include "util/acksync.h"
 #include "util/net.h"
 #include "util/thread.h"
 
@@ -14,10 +15,13 @@ struct receiver {
     sc_socket control_socket;
     sc_thread thread;
     sc_mutex mutex;
+
+    struct sc_acksync *acksync;
 };
 
 bool
-receiver_init(struct receiver *receiver, sc_socket control_socket);
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+              struct sc_acksync *acksync);
 
 void
 receiver_destroy(struct receiver *receiver);