Implement Java-specific ATN data optimization (+-2 shift)

antlr · Feb 6, 2022 · 6040190 · 6040190
1 parent 2ed4948
commit 6040190
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 62 deletions.
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java
@@ -219,7 +219,7 @@ protected ATN createATN(Grammar g, boolean useSerializer) {
 
 		ATN atn = g.atn;
 		if ( useSerializer ) {
-			char[] serialized = ATNSerializer.getSerializedAsChars(atn);
+			char[] serialized = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());
 			return new ATNDeserializer().deserialize(serialized);
 		}
 

diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestInterpreterDataReader.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestInterpreterDataReader.java
@@ -76,7 +76,7 @@ public void testParseFile() throws IOException, NoSuchFieldException, IllegalAcc
 		Assert.assertNull(channels);
 		Assert.assertNull(modes);
 
-        char[] atnChars = ATNSerializer.getSerializedAsChars(atn);
+        char[] atnChars = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());
 		Assert.assertEquals(ATNDeserializer.SERIALIZED_VERSION, atnChars[0]);
     }
 

diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java
@@ -83,6 +83,11 @@ public ATNDeserializer(ATNDeserializationOptions deserializationOptions) {
 	}
 
 	public ATN deserialize(char[] data) {
+		data = data.clone();
+		for (int i = 1; i < data.length; i++) {
+			data[i] = (char) (data[i] - 2);
+		}
+
 		int p = 0;
 		int version = toInt(data[p++]);
 		if (version != SERIALIZED_VERSION) {

diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java
@@ -65,7 +65,7 @@ public ATNSerializer(ATN atn, List<String> tokenNames) {
 	 *
 	 *  Convenient to pack into unsigned shorts to make as Java string.
 	 */
-	public IntegerList serialize() {
+	public IntegerList serialize(String language) {
 		IntegerList data = new IntegerList();
 		data.add(ATNDeserializer.SERIALIZED_VERSION);
 
@@ -354,14 +354,15 @@ public void serializeCodePoint(IntegerList data, int cp) {
 			}
 		}
 
-		for (int i = 0; i < data.size(); i++) {
+		boolean isJava = language.equals("Java");
+		for (int i = 1; i < data.size(); i++) {
 			int value = data.get(i);
 			if (value < Character.MIN_VALUE || value > Character.MAX_VALUE) {
 				throw new UnsupportedOperationException("Serialized ATN data element " +
 						value + " element " + i + " out of range " + (int) Character.MIN_VALUE + ".." + (int) Character.MAX_VALUE);
 			}
 
-			data.set(i, value);
+			data.set(i, isJava ? (value + 2) & 0xFFFF : value);
 		}
 
 		return data;
@@ -404,6 +405,12 @@ private static void serializeSets(
 	}
 
 	public String decode(char[] data) {
+		data = data.clone();
+		// don't adjust the first value since that's the version number
+		for (int i = 1; i < data.length; i++) {
+			data[i] = (char)(data[i] - 2);
+		}
+
 		StringBuilder buf = new StringBuilder();
 		int p = 0;
 		int version = ATNDeserializer.toInt(data[p++]);
@@ -574,22 +581,16 @@ public String getTokenName(int t) {
 	}
 
 	/** Used by Java target to encode short/int array as chars in string. */
-	public static String getSerializedAsString(ATN atn) {
-		return new String(getSerializedAsChars(atn));
-	}
-
-	public static IntegerList getSerialized(ATN atn) {
-		return new ATNSerializer(atn).serialize();
+	public static String getSerializedAsString(ATN atn, String language) {
+		return new String(getSerializedAsChars(atn, language));
 	}
 
-	public static char[] getSerializedAsChars(ATN atn) {
-		return Utils.toCharArray(getSerialized(atn));
+	public static IntegerList getSerialized(ATN atn, String language) {
+		return new ATNSerializer(atn).serialize(language);
 	}
 
-	public static String getDecoded(ATN atn, List<String> tokenNames) {
-		IntegerList serialized = getSerialized(atn);
-		char[] data = Utils.toCharArray(serialized);
-		return new ATNSerializer(atn, tokenNames).decode(data);
+	public static char[] getSerializedAsChars(ATN atn, String language) {
+		return Utils.toCharArray(getSerialized(atn, language));
 	}
 
 	private void serializeInt(IntegerList data, int value) {

diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java
@@ -162,10 +162,10 @@ public void testSetUp() throws Exception {
 
 	protected void checkDeserializationIsStable(Grammar g) {
 		ATN atn = createATN(g, false);
-		char[] data = Utils.toCharArray(ATNSerializer.getSerialized(atn));
-		String atnData = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
+		char[] data = Utils.toCharArray(ATNSerializer.getSerialized(atn, "Java"));
+		String atnData = TestATNSerialization.getDecoded(atn, Arrays.asList(g.getTokenNames()));
 		ATN atn2 = new ATNDeserializer().deserialize(data);
-		String atn2Data = ATNSerializer.getDecoded(atn2, Arrays.asList(g.getTokenNames()));
+		String atn2Data = TestATNSerialization.getDecoded(atn2, Arrays.asList(g.getTokenNames()));
 
 		assertEquals(atnData, atn2Data);
 	}