Skip to content

Commit

Permalink
enhancement(LorieView.java): improve input handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@twaik
twaik committed Dec 11, 2024
1 parent 0d448f7 commit e7fbf10
Show file tree
Hide file tree
Showing 5 changed files with 170 additions and 70 deletions.
29 changes: 26 additions & 3 deletions app/src/main/cpp/lorie/android.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,6 @@ Java_com_termux_x11_CmdEntryPoint_listenForConnections(JNIEnv *env, jobject thiz
}
}
close(client);
client = -1;
}
}

Expand All @@ -589,8 +588,11 @@ static inline void checkConnection(JNIEnv* env) {
}
}

static jobject mLorieView = NULL;
static jmethodID mFlushComposingView = NULL;

JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_connect(__unused JNIEnv* env, __unused jobject cls, jint fd) {
Java_com_termux_x11_LorieView_nativeInit(JNIEnv *env, jobject thiz) {
if (!Charset.self) {
// Init clipboard-related JNI stuff
Charset.self = FindClassOrDie(env, "java/nio/charset/Charset");
Expand All @@ -601,6 +603,19 @@ Java_com_termux_x11_LorieView_connect(__unused JNIEnv* env, __unused jobject cls
CharBuffer.toString = FindMethodOrDie(env, CharBuffer.self, "toString", "()Ljava/lang/String;", JNI_FALSE);
}

mLorieView = (*env)->NewGlobalRef(env, thiz);
mFlushComposingView = (*env)->GetMethodID(env, (*env)->GetObjectClass(env, thiz), "flushComposingView", "()V");
}

static void flushComposingView(JNIEnv* env) {
if (!env || !mLorieView || !mFlushComposingView)
return;

(*env)->CallVoidMethod(env, mLorieView, mFlushComposingView);
}

JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_connect(__unused JNIEnv* env, __unused jobject cls, jint fd) {
conn_fd = fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
checkConnection(env);
Expand Down Expand Up @@ -679,6 +694,7 @@ Java_com_termux_x11_LorieView_setClipboardSyncEnabled(__unused JNIEnv* env, __un
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_sendClipboardAnnounce(JNIEnv *env, __unused jobject thiz) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .type = EVENT_CLIPBOARD_ANNOUNCE };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -688,6 +704,7 @@ Java_com_termux_x11_LorieView_sendClipboardAnnounce(JNIEnv *env, __unused jobjec
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_sendClipboardEvent(JNIEnv *env, __unused jobject thiz, jbyteArray text) {
if (conn_fd != -1 && text) {
flushComposingView(env);
jsize length = (*env)->GetArrayLength(env, text);
jbyte* str = (*env)->GetByteArrayElements(env, text, NULL);
lorieEvent e = { .clipboardSend = { .t = EVENT_CLIPBOARD_SEND, .count = length } };
Expand All @@ -701,6 +718,7 @@ Java_com_termux_x11_LorieView_sendClipboardEvent(JNIEnv *env, __unused jobject t
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_sendWindowChange(__unused JNIEnv* env, __unused jobject cls, jint width, jint height, jint framerate) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .screenSize = { .t = EVENT_SCREEN_SIZE, .width = width, .height = height, .framerate = framerate } };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -710,6 +728,7 @@ Java_com_termux_x11_LorieView_sendWindowChange(__unused JNIEnv* env, __unused jo
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_sendMouseEvent(__unused JNIEnv* env, __unused jobject cls, jfloat x, jfloat y, jint which_button, jboolean button_down, jboolean relative) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .mouse = { .t = EVENT_MOUSE, .x = x, .y = y, .detail = which_button, .down = button_down, .relative = relative } };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -719,6 +738,7 @@ Java_com_termux_x11_LorieView_sendMouseEvent(__unused JNIEnv* env, __unused jobj
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_sendTouchEvent(__unused JNIEnv* env, __unused jobject cls, jint action, jint id, jint x, jint y) {
if (conn_fd != -1 && action != -1) {
flushComposingView(env);
lorieEvent e = { .touch = { .t = EVENT_TOUCH, .type = action, .id = id, .x = x, .y = y } };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -730,6 +750,7 @@ Java_com_termux_x11_LorieView_sendStylusEvent(JNIEnv *env, __unused jobject thiz
jint pressure, jint tilt_x, jint tilt_y,
jint orientation, jint buttons, jboolean eraser, jboolean mouse) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .stylus = { .t = EVENT_STYLUS, .x = x, .y = y, .pressure = pressure, .tilt_x = tilt_x, .tilt_y = tilt_y, .orientation = orientation, .buttons = buttons, .eraser = eraser, .mouse = mouse } };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -739,6 +760,7 @@ Java_com_termux_x11_LorieView_sendStylusEvent(JNIEnv *env, __unused jobject thiz
JNIEXPORT void JNICALL
Java_com_termux_x11_LorieView_requestStylusEnabled(JNIEnv *env, __unused jclass clazz, jboolean enabled) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .stylusEnable = { .t = EVENT_STYLUS_ENABLE, .enable = enabled } };
write(conn_fd, &e, sizeof(e));
checkConnection(env);
Expand All @@ -748,6 +770,7 @@ Java_com_termux_x11_LorieView_requestStylusEnabled(JNIEnv *env, __unused jclass
JNIEXPORT jboolean JNICALL
Java_com_termux_x11_LorieView_sendKeyEvent(__unused JNIEnv* env, __unused jobject cls, jint scan_code, jint key_code, jboolean key_down) {
if (conn_fd != -1) {
flushComposingView(env);
int code = (scan_code) ?: android_to_linux_keycode[key_code];
log(DEBUG, "Sending key: %d (%d %d %d)", code + 8, scan_code, key_code, key_down);
lorieEvent e = { .key = { .t = EVENT_KEY, .key = code + 8, .state = key_down } };
Expand Down Expand Up @@ -788,7 +811,7 @@ Java_com_termux_x11_LorieView_sendTextEvent(JNIEnv *env, __unused jobject thiz,
p += len;
if (p - (char*) str >= length)
break;
usleep(30000);
// usleep(30000);
}

(*env)->ReleaseByteArrayElements(env, text, str, JNI_ABORT);
Expand Down
185 changes: 124 additions & 61 deletions app/src/main/java/com/termux/x11/LorieView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.drawable.ColorDrawable;
import android.os.Build;
import android.text.Editable;
import android.text.InputFilter;
import android.text.InputType;
import android.util.AttributeSet;
import android.util.Log;
Expand All @@ -23,18 +23,20 @@
import android.view.inputmethod.BaseInputConnection;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.InputMethodSubtype;
import android.widget.TextView;

import androidx.annotation.Keep;
import androidx.annotation.NonNull;

import com.termux.x11.input.InputStub;
import com.termux.x11.input.TouchInputHandler;

import java.nio.charset.StandardCharsets;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.regex.PatternSyntaxException;

import static java.nio.charset.StandardCharsets.UTF_8;

@Keep @SuppressLint("WrongConstant")
@SuppressWarnings("deprecation")
public class LorieView extends SurfaceView implements InputStub {
Expand All @@ -50,12 +52,102 @@ interface PixelFormat {
private long lastClipboardTimestamp = System.currentTimeMillis();
private static boolean clipboardSyncEnabled = false;
private static boolean hardwareKbdScancodesWorkaround = false;
private final InputMethodManager mIMM = (InputMethodManager)getContext().getSystemService( Context.INPUT_METHOD_SERVICE);
private String mImeLang;
private boolean mImeCJK;
public boolean enableGboardCJK;
private Callback mCallback;
private final Point p = new Point();
private TextView composingView;
private final Editable mEditable = new Editable() {
// Wrap editable methods and hook and intercept changes
void onChange() {
int len = thiz.length();
char[] chars = new char[len];
thiz.getChars(0, len, chars, 0);
onComposingTextChange(chars, len);
}
private final Editable thiz = Editable.Factory.getInstance().newEditable("");
@Override public int length() { return thiz.length(); }
@Override public char charAt(int index) { return thiz.charAt(index); }
@NonNull @Override public CharSequence subSequence(int start, int end) { return thiz.subSequence(start, end); }
@Override public <T> T[] getSpans(int start, int end, Class<T> type) { return thiz.getSpans(start, end, type); }
@Override public int getSpanStart(Object tag) { return thiz.getSpanStart(tag); }
@Override public int getSpanEnd(Object tag) { return thiz.getSpanEnd(tag); }
@Override public int getSpanFlags(Object tag) { return thiz.getSpanFlags(tag); }
@Override public int nextSpanTransition(int start, int limit, Class type) { return thiz.nextSpanTransition(start, limit, type); }
@Override public void setSpan(Object what, int start, int end, int flags) { thiz.setSpan(what, start, end, flags); onChange(); }
@Override public void removeSpan(Object what) { thiz.removeSpan(what); onChange(); }
@Override public void getChars(int start, int end, char[] dest, int destoff) { thiz.getChars(start, end, dest, destoff); }
@NonNull @Override public Editable replace(int st, int en, CharSequence source, int start, int end) { Editable e = thiz.replace(st, en, source, start, end); onChange(); android.util.Log.d("EDITABLE", "replace0"); return e; }
@NonNull @Override public Editable replace(int st, int en, CharSequence text) { Editable e = thiz.replace(st, en, text); onChange(); android.util.Log.d("EDITABLE", "replace1" + st + " " + en + " " + text); return e; }
@NonNull @Override public Editable insert(int where, CharSequence text, int start, int end) { Editable e = thiz.insert(where, text, start, end); onChange(); android.util.Log.d("EDITABLE", "insert0"); return e; }
@NonNull @Override public Editable insert(int where, CharSequence text) { Editable e = thiz.insert(where, text); onChange(); android.util.Log.d("EDITABLE", "insert1"); return e; }
@NonNull @Override public Editable delete(int st, int en) { Editable e = thiz.delete(st, en); onChange(); android.util.Log.d("EDITABLE", "delete"); return e; }
@NonNull @Override public Editable append(CharSequence text) { Editable e = thiz.append(text); onChange(); android.util.Log.d("EDITABLE", "append0"); return e; }
@NonNull @Override public Editable append(CharSequence text, int start, int end) { Editable e = thiz.append(text, start, end); onChange(); android.util.Log.d("EDITABLE", "append1"); return e; }
@NonNull @Override public Editable append(char text) { Editable e = thiz.append(text); onChange(); android.util.Log.d("EDITABLE", "append2"); return e; }
@Override public void clear() { thiz.clear(); onChange(); }
@Override public void clearSpans() { thiz.clearSpans(); onChange(); }
@Override public void setFilters(InputFilter[] filters) { thiz.setFilters(filters); onChange(); }
@Override public InputFilter[] getFilters() { return thiz.getFilters(); }
};
private final InputConnection mConnection = new BaseInputConnection(this, false) {
private final CharSequence seq = " ";

@Override
public Editable getEditable() {
return mEditable;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public CharSequence getTextBeforeCursor(int length, int flags) {
return seq;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public CharSequence getTextAfterCursor(int length, int flags) {
return seq;
}

@Override
public boolean deleteSurroundingText(int beforeLength, int afterLength) {
for (int i=0; i<beforeLength; i++) {
LorieView.this.sendKeyEvent(0, KeyEvent.KEYCODE_DEL, true);
LorieView.this.sendKeyEvent(0, KeyEvent.KEYCODE_DEL, false);
}
for (int i=0; i<afterLength; i++) {
LorieView.this.sendKeyEvent(0, KeyEvent.KEYCODE_FORWARD_DEL, true);
LorieView.this.sendKeyEvent(0, KeyEvent.KEYCODE_FORWARD_DEL, false);
}
return true;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public boolean setComposingRegion(int start, int end) {
return true;
}

@Override
public boolean commitText(CharSequence text, int newCursorPosition) {
sendTextEvent(text.toString().getBytes(UTF_8));
return true;
}

@Override
public boolean finishComposingText() {
synchronized (mEditable) {
if (mEditable.length() <= 0)
return true;

char[] t = new char[mEditable.length()];
mEditable.getChars(0, mEditable.length(), t, 0);
mEditable.clear();
sendTextEvent(UTF_8.encode(CharBuffer.wrap(t)).array());
}

return true;
}
};
private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() {
@Override public void surfaceCreated(@NonNull SurfaceHolder holder) {
holder.setFormat(PixelFormat.BGRA_8888);
Expand Down Expand Up @@ -88,6 +180,7 @@ interface PixelFormat {
private void init() {
getHolder().addCallback(mSurfaceCallback);
clipboard = (ClipboardManager) getContext().getSystemService(Context.CLIPBOARD_SERVICE);
nativeInit();
}

public void setCallback(Callback callback) {
Expand Down Expand Up @@ -235,8 +328,6 @@ public void reloadPreferences(Prefs p) {
clipboardSyncEnabled = p.clipboardEnable.get();
setClipboardSyncEnabled(clipboardSyncEnabled, clipboardSyncEnabled);
TouchInputHandler.refreshInputDevices();
enableGboardCJK = p.enableGboardCJK.get();
mIMM.restartInput(this);
}

// It is used in native code
Expand All @@ -252,14 +343,14 @@ void setClipboardText(String text) {
/** @noinspection unused*/ // It is used in native code
void requestClipboard() {
if (!clipboardSyncEnabled) {
sendClipboardEvent("".getBytes(StandardCharsets.UTF_8));
sendClipboardEvent("".getBytes(UTF_8));
return;
}

CharSequence clip = clipboard.getText();
if (clip != null) {
String text = String.valueOf(clipboard.getText());
sendClipboardEvent(text.getBytes(StandardCharsets.UTF_8));
sendClipboardEvent(text.getBytes(UTF_8));
Log.d("CLIP", "sending clipboard contents: " + text);
}
}
Expand Down Expand Up @@ -298,69 +389,41 @@ public void onWindowFocusChanged(boolean hasFocus) {
TouchInputHandler.refreshInputDevices();
}

public void checkRestartInput(boolean recheck) {
if (!enableGboardCJK)
public void setComposingView(TextView v) {
composingView = v;
}

private void onComposingTextChange(char[] chars, int len) {
if (composingView == null)
return;

InputMethodSubtype methodSubtype = mIMM.getCurrentInputMethodSubtype();
String languageTag = methodSubtype == null ? null : methodSubtype.getLanguageTag();
if (languageTag != null && languageTag.length() >= 2 && !languageTag.substring(0, 2).equals(mImeLang))
mIMM.restartInput(this);
else if (recheck) { // recheck needed because sometimes requestCursorUpdates() is called too fast, before InputMethodManager detect change in IM subtype
MainActivity.handler.postDelayed(() -> checkRestartInput(false), 40);
}
composingView.setText(String.format(" %s ", new String(chars, 0, len)));
composingView.setVisibility(len == 0 ? LorieView.INVISIBLE : LorieView.VISIBLE);
}

@Override
public InputConnection onCreateInputConnection(EditorInfo outAttrs) {
outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS | InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD;
outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS | InputType.TYPE_TEXT_VARIATION_NORMAL;
outAttrs.actionLabel = "↵";

// Note that IME_ACTION_NONE cannot be used as that makes it impossible to input newlines using the on-screen
// keyboard on Android TV (see https://github.com/termux/termux-app/issues/221).
outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_FULLSCREEN;
return mConnection;
}

if (enableGboardCJK) {
InputMethodSubtype methodSubtype = mIMM.getCurrentInputMethodSubtype();
mImeLang = methodSubtype == null ? null : methodSubtype.getLanguageTag();
if (mImeLang != null && mImeLang.length() > 2)
mImeLang = mImeLang.substring(0, 2);
mImeCJK = mImeLang != null && (mImeLang.equals("zh") || mImeLang.equals("ko") || mImeLang.equals("ja"));
outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS |
(mImeCJK ? InputType.TYPE_TEXT_VARIATION_NORMAL : InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD);
return new BaseInputConnection(this, false) {
// workaround for Gboard
// Gboard calls requestCursorUpdates() whenever switching language
// check and then restart keyboard in different inputType when needed
@Override
public Editable getEditable() {
checkRestartInput(true);
return super.getEditable();
}
@Override
public boolean requestCursorUpdates(int cursorUpdateMode) {
checkRestartInput(true);
return super.requestCursorUpdates(cursorUpdateMode);
}

@Override
public boolean commitText(CharSequence text, int newCursorPosition) {
boolean result = super.commitText(text, newCursorPosition);
if (mImeCJK)
// suppress Gboard CJK keyboard suggestion
// this workaround does not work well for non-CJK keyboards
// , when typing fast and two keypresses (commitText) are close in time
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU)
mIMM.invalidateInput(LorieView.this);
else
mIMM.restartInput(LorieView.this);
return result;
}
};
} else {
return super.onCreateInputConnection(outAttrs);
// Used in native method
/** @noinspection unused*/
private void flushComposingView() {
synchronized (mEditable) {
if (mEditable.length() <= 0)
return;
}

mConnection.finishComposingText();
}

private native void nativeInit();
static native void connect(int fd);
native void handleXEvents();
static native void startLogcat(int fd);
Expand Down
Loading

0 comments on commit e7fbf10

Please sign in to comment.