Fix #483: Umlauts and other diacritics cause wrong line wrap

RSyntaxTextArea/src/main/java/org/fife/ui/rsyntaxtextarea/modes/PlainTextTokenMaker.flex

@@ -200,9 +200,9 @@ import org.fife.ui.rsyntaxtextarea.*;
 
 %}
 
-LetterOrDigit	= ([a-zA-Z0-9])
+LetterOrDigit	= ([[:letter:][:digit:]])
 Identifier		= ({LetterOrDigit}+)
-Separator		= ([^a-zA-Z0-9 \t\n])
+Separator		= ([^[:letter:][:digit:] \t\n])
 WhiteSpace		= ([ \t]+)
 LineTerminator	= ([\n])
 

RSyntaxTextArea/src/test/java/org/fife/ui/rsyntaxtextarea/modes/AbstractTokenMakerTest.java

@@ -74,6 +74,9 @@ protected void assertAllTokensOfType(int expectedType, int initialTokenType,
 
 			Assertions.assertEquals(expectedType, t.getType(),
 				"Token has unexpected type: orig=" + token + ", actual=" + t);
+			// The assertion below doesn't work as some tests have trailing '\n' chars
+			//Assertions.assertEquals(token, t.getLexeme(),
+			//	"Token has unexpected lexeme: expected='" + token + "', actual=" + t);
 
 			// The token array passed to this method is supposed to be single tokens.
 			// The next token should denote it's the end of a line, and what the next
@@ -86,7 +89,7 @@ protected void assertAllTokensOfType(int expectedType, int initialTokenType,
 						!t.getNextToken().isPaintable() ||
 						// 3. The next token is a token type to continue, e.g. string or MLC
 						t.getNextToken().length() == 0,
-					"Next token does not denote end-of-line: " + t.getNextToken());
+					"Next token does not denote end-of-line! lexeme: " + t.getLexeme() + ", next: " + t.getNextToken());
 			}
 		}
 

RSyntaxTextArea/src/test/java/org/fife/ui/rsyntaxtextarea/modes/PlainTextTokenMakerTest.java

@@ -76,7 +76,8 @@ void testIdentifiers() {
 			"foo",
 			"foo999",
 			"912",
-			"FooBar"
+			"FooBar",
+			"unn\u00f6tig"
 		);
 	}