From e2c258a93cafd5c7e8468e13c984e2199953d986 Mon Sep 17 00:00:00 2001
From: kit <kitbdev@gmail.com>
Date: Tue, 2 Jul 2024 00:02:37 -0400
Subject: [PATCH] Treat punctuation as words and as word separators

---
 servers/text_server.cpp      |  37 ++++-
 tests/scene/test_text_edit.h | 265 ++++++++++++++++++++++++++++++++++-
 2 files changed, 292 insertions(+), 10 deletions(-)

diff --git a/servers/text_server.cpp b/servers/text_server.cpp
index f391c795142b..e72c2c119077 100644
--- a/servers/text_server.cpp
+++ b/servers/text_server.cpp
@@ -1104,20 +1104,45 @@ PackedInt32Array TextServer::shaped_text_get_word_breaks(const RID &p_shaped, Bi
 	const_cast<TextServer *>(this)->shaped_text_update_justification_ops(p_shaped);
 	const Vector2i &range = shaped_text_get_range(p_shaped);
 
-	int word_start = range.x;
-
 	const int l_size = shaped_text_get_glyph_count(p_shaped);
 	const Glyph *l_gl = const_cast<TextServer *>(this)->shaped_text_sort_logical(p_shaped);
 
+	int word_start = range.x;
+	int word_end = l_size > 0 ? l_gl[0].start : 0;
+	bool was_whitespace = true;
+	bool was_punctuation = (p_skip_grapheme_flags & GRAPHEME_IS_SPACE) != 0;
+
 	for (int i = 0; i < l_size; i++) {
 		if (l_gl[i].count > 0) {
-			if ((l_gl[i].flags & p_grapheme_flags) != 0 && (l_gl[i].flags & p_skip_grapheme_flags) == 0) {
-				int next = (i == 0) ? l_gl[i].start : l_gl[i - 1].end;
-				if (word_start < next) {
+			if ((l_gl[i].flags & p_skip_grapheme_flags) != 0) {
+				continue;
+			}
+			if ((l_gl[i].flags & GRAPHEME_IS_SPACE) != 0) {
+				if (!was_whitespace) {
 					words.push_back(word_start);
-					words.push_back(next);
+					words.push_back(word_end);
 				}
+				was_whitespace = true;
+				was_punctuation = false;
 				word_start = l_gl[i].end;
+			} else if ((l_gl[i].flags & p_grapheme_flags) != 0) {
+				if (!was_punctuation && !was_whitespace) {
+					words.push_back(word_start);
+					words.push_back(word_end);
+					word_start = l_gl[i].start;
+				}
+				was_whitespace = false;
+				was_punctuation = true;
+				word_end = l_gl[i].end;
+			} else {
+				if (was_punctuation) {
+					words.push_back(word_start);
+					words.push_back(word_end);
+					word_start = l_gl[i].start;
+				}
+				was_whitespace = false;
+				was_punctuation = false;
+				word_end = l_gl[i].end;
 			}
 		}
 	}
diff --git a/tests/scene/test_text_edit.h b/tests/scene/test_text_edit.h
index 69e27fe7a086..f1f24c042398 100644
--- a/tests/scene/test_text_edit.h
+++ b/tests/scene/test_text_edit.h
@@ -1358,6 +1358,62 @@ TEST_CASE("[SceneTree][TextEdit] text entry") {
 			SIGNAL_CHECK_FALSE("caret_changed");
 		}
 
+		SUBCASE("[TextEdit] select words in other languages") {
+			text_edit->set_text(U"سلسلة الاختبار");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"سلسلة");
+			CHECK(text_edit->get_caret_column() == 5);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+
+			text_edit->set_text(U"מחרוזת בדיקה");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"מחרוזת");
+			CHECK(text_edit->get_caret_column() == 6);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+
+			text_edit->set_text(U"测试 字符串");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"测试");
+			CHECK(text_edit->get_caret_column() == 2);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+
+			text_edit->set_text(U"テスト 文学列");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"テスト");
+			CHECK(text_edit->get_caret_column() == 3);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+
+			text_edit->set_text(U"테스트 문자열");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"테스트");
+			CHECK(text_edit->get_caret_column() == 3);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+
+			text_edit->set_text(U"👏👏👏 👏👏");
+			text_edit->set_caret_column(1);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_selected_text() == U"👏👏👏");
+			CHECK(text_edit->get_caret_column() == 3);
+			CHECK(text_edit->get_selection_origin_column() == 0);
+			text_edit->deselect();
+		}
+
 		SUBCASE("[TextEdit] add selection for next occurrence") {
 			text_edit->set_text("\ntest   other_test\nrandom   test\nword test word nonrandom");
 			text_edit->set_caret_column(0);
@@ -4153,7 +4209,7 @@ TEST_CASE("[SceneTree][TextEdit] text entry") {
 			text_edit->start_action(TextEdit::ACTION_NONE);
 
 			// Remove text to the start of the word to the left of the caret.
-			text_edit->set_caret_column(text_edit->get_line(0).length());
+			text_edit->set_caret_column(15);
 			text_edit->set_caret_column(12, false, 1);
 			MessageQueue::get_singleton()->flush();
 			SIGNAL_DISCARD("caret_changed");
@@ -4161,7 +4217,7 @@ TEST_CASE("[SceneTree][TextEdit] text entry") {
 
 			SEND_GUI_ACTION("ui_text_backspace_word");
 			CHECK(text_edit->get_viewport()->is_input_handled());
-			CHECK(text_edit->get_text() == "this is st \nthis ime t text.");
+			CHECK(text_edit->get_text() == "this is st .\nthis ime t text.");
 			CHECK(text_edit->get_caret_count() == 2);
 			CHECK_FALSE(text_edit->has_selection(0));
 			CHECK(text_edit->get_caret_line(0) == 0);
@@ -4180,7 +4236,7 @@ TEST_CASE("[SceneTree][TextEdit] text entry") {
 			CHECK(text_edit->get_caret_count() == 2);
 			CHECK_FALSE(text_edit->has_selection(0));
 			CHECK(text_edit->get_caret_line(0) == 0);
-			CHECK(text_edit->get_caret_column(0) == 16);
+			CHECK(text_edit->get_caret_column(0) == 15);
 			CHECK_FALSE(text_edit->has_selection(1));
 			CHECK(text_edit->get_caret_line(1) == 1);
 			CHECK(text_edit->get_caret_column(1) == 12);
@@ -4191,7 +4247,7 @@ TEST_CASE("[SceneTree][TextEdit] text entry") {
 			// Redo.
 			text_edit->redo();
 			MessageQueue::get_singleton()->flush();
-			CHECK(text_edit->get_text() == "this is st \nthis ime t text.");
+			CHECK(text_edit->get_text() == "this is st .\nthis ime t text.");
 			CHECK(text_edit->get_caret_count() == 2);
 			CHECK_FALSE(text_edit->has_selection(0));
 			CHECK(text_edit->get_caret_line(0) == 0);
@@ -7338,6 +7394,207 @@ TEST_CASE("[SceneTree][TextEdit] line wrapping") {
 	memdelete(text_edit);
 }
 
+TEST_CASE("[SceneTree][TextEdit] word separators") {
+	TextEdit *text_edit = memnew(TextEdit);
+	SceneTree::get_singleton()->get_root()->add_child(text_edit);
+	text_edit->grab_focus();
+
+	SUBCASE("[TextEdit] common word separators") {
+		// Common separators.
+		String test_separators = " \t.,!=-+*/:\"'`<>()[]%$#";
+		test_separators += (char32_t)0x3000; // CJK space.
+		for (int i = 0; i < test_separators.length(); i++) {
+			char32_t sep = test_separators[i];
+			text_edit->set_text(vformat("test%cword%cseparator", sep, sep));
+			text_edit->set_caret_column(7);
+			text_edit->select_word_under_caret();
+			CHECK(text_edit->has_selection());
+			CHECK(text_edit->get_caret_column() == 9);
+			CHECK(text_edit->get_selection_origin_column() == 5);
+			text_edit->deselect();
+		}
+
+		// Underscore `_` is not a separator.
+		text_edit->set_text("test_word_separator");
+		text_edit->set_caret_column(7);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_caret_column() == 19);
+		CHECK(text_edit->get_selection_origin_column() == 0);
+		text_edit->deselect();
+	}
+
+	SUBCASE("[TextEdit] custom separators") {
+		// Custom word separator underscore `_`.
+		text_edit->set_use_custom_word_separators(true);
+		text_edit->set_custom_word_separators("_");
+		text_edit->set_text("test_word_separator");
+		text_edit->set_caret_column(7);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_caret_column() == 9);
+		CHECK(text_edit->get_selection_origin_column() == 5);
+		text_edit->deselect();
+
+		// Disable custom word separators.
+		text_edit->set_use_custom_word_separators(false);
+		text_edit->set_caret_column(7);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_caret_column() == 19);
+		CHECK(text_edit->get_selection_origin_column() == 0);
+		text_edit->deselect();
+
+		// Custom separators can be disabled default separators.
+		text_edit->set_use_default_word_separators(false);
+		text_edit->set_text("test>word>separator");
+		text_edit->set_caret_column(7);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_caret_column() == 19);
+		CHECK(text_edit->get_selection_origin_column() == 0);
+		text_edit->deselect();
+
+		text_edit->set_use_custom_word_separators(true);
+		text_edit->set_custom_word_separators(">");
+		text_edit->set_caret_column(7);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_caret_column() == 9);
+		CHECK(text_edit->get_selection_origin_column() == 5);
+		text_edit->deselect();
+	}
+
+	SUBCASE("[TextEdit] punctuation as word") {
+		// Punctuation with spaces around it.
+		text_edit->set_text("test != separator");
+		text_edit->set_caret_column(6);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_selected_text() == "!=");
+		CHECK(text_edit->get_caret_column() == 7);
+		CHECK(text_edit->get_selection_origin_column() == 5);
+		text_edit->deselect();
+
+		// Punctuation without spaces around it.
+		text_edit->set_text("test!=separator");
+		text_edit->set_caret_column(5);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_selected_text() == "!=");
+		CHECK(text_edit->get_caret_column() == 6);
+		CHECK(text_edit->get_selection_origin_column() == 4);
+		text_edit->deselect();
+
+		// Another punctuation without spaces around it.
+		text_edit->set_text("test>=separator");
+		text_edit->set_caret_column(5);
+		text_edit->select_word_under_caret();
+		CHECK(text_edit->has_selection());
+		CHECK(text_edit->get_selected_text() == ">=");
+		CHECK(text_edit->get_caret_column() == 6);
+		CHECK(text_edit->get_selection_origin_column() == 4);
+		text_edit->deselect();
+	}
+
+	SUBCASE("[TextEdit] moving over punctuation and spaces") {
+		// Punctuation is used to separate words and can be considered a word itself.
+		text_edit->set_text("test = separator");
+		text_edit->set_caret_column(0);
+		// Move after 'test'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+		// Move after '='.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 6);
+		// Move after 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 16);
+		// Move before 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 7);
+		// Move before '='.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 5);
+
+		text_edit->set_text("test=separator");
+		text_edit->set_caret_column(0);
+		// Move after 'test'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+		// Move after '='.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 5);
+		// Move after 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 14);
+		// Move before 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 5);
+		// Move before '='.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+
+		text_edit->set_text("test= separator");
+		text_edit->set_caret_column(0);
+		// Move after 'test'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+		// Move after '='.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 5);
+		// Move after 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 15);
+		// Move before 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 6);
+		// Move before '='.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+
+		text_edit->set_text("test =separator");
+		text_edit->set_caret_column(0);
+		// Move after 'test'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 4);
+		// Move after '='.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 6);
+		// Move after 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_right");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 15);
+		// Move before 'separator'.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 6);
+		// Move before '='.
+		SEND_GUI_ACTION("ui_text_caret_word_left");
+		CHECK(text_edit->get_viewport()->is_input_handled());
+		CHECK(text_edit->get_caret_column() == 5);
+	}
+
+	memdelete(text_edit);
+}
+
 TEST_CASE("[SceneTree][TextEdit] viewport") {
 	TextEdit *text_edit = memnew(TextEdit);
 	SceneTree::get_singleton()->get_root()->add_child(text_edit);