From a8378d3a7b5dcae470d0f58338891e5ed1b65979 Mon Sep 17 00:00:00 2001 From: Ivan Kochurkin Date: Mon, 7 Feb 2022 17:40:19 +0300 Subject: [PATCH] Support of full int range in serializer and deserializer (up to Integer.MAX_VALUE) fix #840, fix #1863, fix #2732, fix #3338 --- .../v4/test/runtime/BaseRuntimeTest.java | 5 +- .../runtime/GeneratedLexerDescriptors.java | 52 +++- .../antlr/v4/runtime/atn/ATNDataReader.java | 16 +- .../antlr/v4/runtime/atn/ATNDataWriter.java | 36 ++- .../antlr/v4/runtime/atn/ATNDeserializer.java | 112 ++++----- .../antlr/v4/runtime/atn/ATNSerializer.java | 229 +++++++----------- .../antlr/v4/runtime/atn/ATNSimulator.java | 15 -- .../v4/test/tool/TestATNDeserialization.java | 52 +++- .../v4/test/tool/TestATNSerialization.java | 8 +- 9 files changed, 290 insertions(+), 235 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java index f0546226fd..fdd476a6f8 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java @@ -366,9 +366,10 @@ public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(String group, St } if (group.equals("LexerExec")) { - descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfTest(targetName)); - descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfTest(targetName)); + descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfDescriptor(targetName)); + descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfDescriptor(targetName)); descriptors.add(GeneratedLexerDescriptors.getLargeLexerDescriptor(targetName)); + descriptors.add(GeneratedLexerDescriptors.getAtnStatesSizeMoreThan65535Descriptor(targetName)); } return descriptors.toArray(new RuntimeTestDescriptor[0]); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java index e0a1bb5ea3..a4327e5f30 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java @@ -1,7 +1,9 @@ package org.antlr.v4.test.runtime; +import java.util.Collections; + public class GeneratedLexerDescriptors { - static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) { + static RuntimeTestDescriptor getLineSeparatorLfDescriptor(String targetName) { UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor(); result.name = "LineSeparatorLf"; result.targetName = targetName; @@ -20,7 +22,7 @@ static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) { return result; } - static RuntimeTestDescriptor getLineSeparatorCrLfTest(String targetName) { + static RuntimeTestDescriptor getLineSeparatorCrLfDescriptor(String targetName) { UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor(); result.name = "LineSeparatorCrLf"; result.targetName = targetName; @@ -65,4 +67,50 @@ static RuntimeTestDescriptor getLargeLexerDescriptor(String targetName) { "[@1,5:4='',<-1>,1:5]\n"; return result; } + + static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor(String targetName) { + UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor(); + result.name = "AtnStatesSizeMoreThan65535"; + result.notes = "Regression for https://github.com/antlr/antlr4/issues/1863"; + result.targetName = targetName; + result.testType = "Lexer"; + + final int tokensCount = 1024; + final String suffix = String.join("", Collections.nCopies(70, "_")); + + String grammarName = "L"; + StringBuilder grammar = new StringBuilder(); + grammar.append("lexer grammar ").append(grammarName).append(";\n"); + grammar.append('\n'); + StringBuilder input = new StringBuilder(); + StringBuilder output = new StringBuilder(); + int startOffset; + int stopOffset = -2; + for (int i = 0; i < tokensCount; i++) { + String value = "T_" + i + suffix; + grammar.append(value).append(": '").append(value).append("';\n"); + input.append(value).append('\n'); + + startOffset = stopOffset + 2; + stopOffset += value.length() + 1; + + output.append("[@").append(i).append(',').append(startOffset).append(':').append(stopOffset) + .append("='").append(value).append("',<").append(i + 1).append(">,").append(i + 1) + .append(":0]\n"); + } + + grammar.append("\n"); + grammar.append("WS: [ \\t\\r\\n]+ -> skip;\n"); + + startOffset = stopOffset + 2; + stopOffset = startOffset - 1; + output.append("[@").append(tokensCount).append(',').append(startOffset).append(':').append(stopOffset) + .append("='',<-1>,").append(tokensCount + 1).append(":0]\n"); + + result.grammar = grammar.toString(); + result.grammarName = grammarName; + result.input = input.toString(); + result.output = output.toString(); + return result; + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataReader.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataReader.java index d758f88c89..059ff7dce9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataReader.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataReader.java @@ -10,7 +10,21 @@ public ATNDataReader(char[] data) { this.data = data; } - public int readUInt32() { + public int read() { + int value = readUInt16(); + if (value == 0xFFFF) { + return -1; + } + + int mask = value >> ATNDataWriter.MaskBits & 0b11; + return mask == 0 + ? value + : mask == 0b01 + ? (readUInt16() << ATNDataWriter.MaskBits) | (value & ((1 << ATNDataWriter.MaskBits) - 1)) + : readInt32(); + } + + public int readInt32() { return readUInt16() | (readUInt16() << 16); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataWriter.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataWriter.java index c6af554150..0517cd4b8e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataWriter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDataWriter.java @@ -3,19 +3,49 @@ import org.antlr.v4.runtime.misc.IntegerList; public class ATNDataWriter { + public static final int MaskBits = 14; public static final int JavaOptimizeOffset = 2; private final IntegerList data; - private final String language; private final boolean isJava; public ATNDataWriter(IntegerList data, String language) { this.data = data; - this.language = language; this.isJava = language.equals("Java"); } - public void writeUInt32(int value) { + /* Write int of full range [Integer.MIN_VALUE..Integer.MAX_VALUE] in compact format + | encoding | count | type | + | ----------------------------------------------------------- | ----- | ------------ | + | 00xx xxxx xxxx xxxx | 1 | int (14 bit) | + | 01xx xxxx xxxx xxxx xxxx xxxx xxxx xxxx | 2 | int (30 bit) | + | 1000 0000 0000 0000 xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx | 3 | int (32 bit) | + | 1111 1111 1111 1111 | 1 | -1 (0xFFFF) | + */ + public int write(int value) { + if (value == -1) { + writeUInt16(0xFFFF); + return 1; + } + + if (value >= 0) { + if (value < 1 << MaskBits) { + writeUInt16(value); + return 1; + } + else if (value < 1 << (MaskBits + 16)) { + writeUInt16(value & ((1 << MaskBits) - 1) | 0b01 << MaskBits); + writeUInt16(value >>> MaskBits); + return 2; + } + } + + writeUInt16(0b10 << MaskBits); + writeInt32(value); + return 3; + } + + public void writeInt32(int value) { writeUInt16((char)value); writeUInt16((char)(value >> 16)); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java index 69ac84cca8..41fe61bb51 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java @@ -45,8 +45,8 @@ public ATN deserialize(char[] data) { throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason)); } - ATNType grammarType = ATNType.values()[reader.readUInt16()]; - int maxTokenType = reader.readUInt16(); + ATNType grammarType = ATNType.values()[reader.read()]; + int maxTokenType = reader.read(); ATN atn = new ATN(grammarType, maxTokenType); // @@ -54,27 +54,24 @@ public ATN deserialize(char[] data) { // List> loopBackStateNumbers = new ArrayList<>(); List> endStateNumbers = new ArrayList<>(); - int nstates = reader.readUInt16(); + int nstates = reader.read(); for (int i=0; i((LoopEndState) s, loopBackStateNumber)); } else if (s instanceof BlockStartState) { - int endStateNumber = reader.readUInt16(); + int endStateNumber = reader.read(); endStateNumbers.add(new Pair<>((BlockStartState) s, endStateNumber)); } atn.addState(s); @@ -89,37 +86,33 @@ else if (s instanceof BlockStartState) { pair.a.endState = (BlockEndState)atn.states.get(pair.b); } - int numNonGreedyStates = reader.readUInt16(); + int numNonGreedyStates = reader.read(); for (int i = 0; i < numNonGreedyStates; i++) { - int stateNumber = reader.readUInt16(); + int stateNumber = reader.read(); ((DecisionState)atn.states.get(stateNumber)).nonGreedy = true; } - int numPrecedenceStates = reader.readUInt16(); + int numPrecedenceStates = reader.read(); for (int i = 0; i < numPrecedenceStates; i++) { - int stateNumber = reader.readUInt16(); + int stateNumber = reader.read(); ((RuleStartState)atn.states.get(stateNumber)).isLeftRecursiveRule = true; } // // RULES // - int nrules = reader.readUInt16(); + int nrules = reader.read(); if ( atn.grammarType == ATNType.LEXER ) { atn.ruleToTokenType = new int[nrules]; } atn.ruleToStartState = new RuleStartState[nrules]; for (int i=0; i sets = new ArrayList<>(); - // First, read all sets with 16-bit Unicode code points <= U+FFFF. - deserializeSets(reader, sets, ATNSerializer.UnicodeSerializeMode.UNICODE_BMP); - - // Next, deserialize sets with 32-bit arguments <= U+10FFFF. - deserializeSets(reader, sets, ATNSerializer.UnicodeSerializeMode.UNICODE_SMP); + // Read all sets with 16-bit or 32-bit Unicode code points + IntervalSet[] sets = deserializeSets(reader); // // EDGES // - int nedges = reader.readUInt16(); + int nedges = reader.read(); for (int i=0; i sets, ATNSerializer.UnicodeSerializeMode mode) { - int nsets = reader.readUInt16(); + private IntervalSet[] deserializeSets(ATNDataReader reader) { + int nsets = reader.read(); + IntervalSet[] sets = new IntervalSet[nsets]; for (int i=0; i sets) + IntervalSet[] sets) { ATNState target = atn.states.get(trg); switch (type) { @@ -513,8 +487,8 @@ protected Transition edgeFactory(ATN atn, } case Transition.ACTION : return new ActionTransition(target, arg1, arg2, arg3 != 0); - case Transition.SET : return new SetTransition(target, sets.get(arg1)); - case Transition.NOT_SET : return new NotSetTransition(target, sets.get(arg1)); + case Transition.SET : return new SetTransition(target, sets[arg1]); + case Transition.NOT_SET : return new NotSetTransition(target, sets[arg1]); case Transition.WILDCARD : return new WildcardTransition(target); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java index f4fa12230c..b9bc7f23ee 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java @@ -13,20 +13,9 @@ import org.antlr.v4.runtime.misc.Utils; import java.io.InvalidClassException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; public class ATNSerializer { - enum UnicodeSerializeMode { - UNICODE_BMP, - UNICODE_SMP - } - public final ATN atn; private final List tokenNames; @@ -74,22 +63,19 @@ public IntegerList serialize(String language) { writer.writeUInt16(ATNDeserializer.SERIALIZED_VERSION, false); // convert grammar type to ATN const to avoid dependence on ANTLRParser - writer.writeUInt16(atn.grammarType.ordinal()); - writer.writeUInt16(atn.maxTokenType); + writer.write(atn.grammarType.ordinal()); + writer.write(atn.maxTokenType); int nedges = 0; - // Note that we use a LinkedHashMap as a set to - // maintain insertion order while deduplicating - // entries with the same key. - Map sets = new LinkedHashMap<>(); + Set sets = new HashSet<>(); // dump states, count edges and collect sets while doing so IntegerList nonGreedyStates = new IntegerList(); IntegerList precedenceStates = new IntegerList(); - writer.writeUInt16(atn.states.size()); + writer.write(atn.states.size()); for (ATNState s : atn.states) { if ( s==null ) { // might be optimized away - writer.writeUInt16(ATNState.INVALID_TYPE); + writer.write(ATNState.INVALID_TYPE); continue; } @@ -102,15 +88,14 @@ public IntegerList serialize(String language) { precedenceStates.add(s.stateNumber); } - writer.writeUInt16(stateType); - - writer.writeUInt16(s.ruleIndex == -1 ? Character.MAX_VALUE : s.ruleIndex); + writer.write(stateType); + writer.write(s.ruleIndex); if ( s.getStateType() == ATNState.LOOP_END ) { - writer.writeUInt16(((LoopEndState)s).loopBackState.stateNumber); + writer.write(((LoopEndState)s).loopBackState.stateNumber); } else if ( s instanceof BlockStartState ) { - writer.writeUInt16(((BlockStartState)s).endState.stateNumber); + writer.write(((BlockStartState)s).endState.stateNumber); } if (s.getStateType() != ATNState.RULE_STOP) { @@ -123,58 +108,50 @@ else if ( s instanceof BlockStartState ) { int edgeType = Transition.serializationTypes.get(t.getClass()); if ( edgeType == Transition.SET || edgeType == Transition.NOT_SET ) { SetTransition st = (SetTransition)t; - sets.put(st.set, true); + sets.add(st.set); } } } // non-greedy states - writer.writeUInt16(nonGreedyStates.size()); + writer.write(nonGreedyStates.size()); for (int i = 0; i < nonGreedyStates.size(); i++) { - writer.writeUInt16(nonGreedyStates.get(i)); + writer.write(nonGreedyStates.get(i)); } // precedence states - writer.writeUInt16(precedenceStates.size()); + writer.write(precedenceStates.size()); for (int i = 0; i < precedenceStates.size(); i++) { - writer.writeUInt16(precedenceStates.get(i)); + writer.write(precedenceStates.get(i)); } int nrules = atn.ruleToStartState.length; - writer.writeUInt16(nrules); + writer.write(nrules); for (int r=0; r0 ) { for (ATNState modeStartState : atn.modeToStartState) { - writer.writeUInt16(modeStartState.stateNumber); + writer.write(modeStartState.stateNumber); } } - List bmpSets = new ArrayList<>(); - List smpSets = new ArrayList<>(); - for (IntervalSet set : sets.keySet()) { - List localSets = !set.isNil() && set.getMaxElement() <= Character.MAX_VALUE ? bmpSets : smpSets; - localSets.add(set); - } - serializeSets(writer, bmpSets, UnicodeSerializeMode.UNICODE_BMP); - serializeSets(writer, smpSets, UnicodeSerializeMode.UNICODE_SMP); + + serializeSets(writer, sets); + Map setIndices = new HashMap<>(); int setIndex = 0; - for (IntervalSet bmpSet : bmpSets) { - setIndices.put(bmpSet, setIndex++); - } - for (IntervalSet smpSet : smpSets) { - setIndices.put(smpSet, setIndex++); + for (IntervalSet set : sets) { + setIndices.put(set, setIndex++); } - writer.writeUInt16(nedges); + writer.write(nedges); for (ATNState s : atn.states) { if ( s==null ) { // might be optimized away @@ -236,10 +213,6 @@ else if ( s instanceof BlockStartState ) { ActionTransition at = (ActionTransition)t; arg1 = at.ruleIndex; arg2 = at.actionIndex; - if (arg2 == -1) { - arg2 = 0xFFFF; - } - arg3 = at.isCtxDependent ? 1 : 0 ; break; case Transition.SET : @@ -250,65 +223,65 @@ else if ( s instanceof BlockStartState ) { break; } - writer.writeUInt16(src); - writer.writeUInt16(trg); - writer.writeUInt16(edgeType); - writer.writeUInt16(arg1); - writer.writeUInt16(arg2); - writer.writeUInt16(arg3); + writer.write(src); + writer.write(trg); + writer.write(edgeType); + writer.write(arg1); + writer.write(arg2); + writer.write(arg3); } } int ndecisions = atn.decisionToState.size(); - writer.writeUInt16(ndecisions); + writer.write(ndecisions); for (DecisionState decStartState : atn.decisionToState) { - writer.writeUInt16(decStartState.stateNumber); + writer.write(decStartState.stateNumber); } // // LEXER ACTIONS // if (atn.grammarType == ATNType.LEXER) { - writer.writeUInt16(atn.lexerActions.length); + writer.write(atn.lexerActions.length); for (LexerAction action : atn.lexerActions) { - writer.writeUInt16(action.getActionType().ordinal()); + writer.write(action.getActionType().ordinal()); switch (action.getActionType()) { case CHANNEL: int channel = ((LexerChannelAction)action).getChannel(); - writer.writeUInt16(channel != -1 ? channel : 0xFFFF); - writer.writeUInt16(0); + writer.write(channel); + writer.write(0); break; case CUSTOM: int ruleIndex = ((LexerCustomAction)action).getRuleIndex(); int actionIndex = ((LexerCustomAction)action).getActionIndex(); - writer.writeUInt16(ruleIndex != -1 ? ruleIndex : 0xFFFF); - writer.writeUInt16(actionIndex != -1 ? actionIndex : 0xFFFF); + writer.write(ruleIndex); + writer.write(actionIndex); break; case MODE: int mode = ((LexerModeAction)action).getMode(); - writer.writeUInt16(mode != -1 ? mode : 0xFFFF); - writer.writeUInt16(0); + writer.write(mode); + writer.write(0); break; case MORE: case POP_MODE: case SKIP: - writer.writeUInt16(0); - writer.writeUInt16(0); + writer.write(0); + writer.write(0); break; case PUSH_MODE: mode = ((LexerPushModeAction)action).getMode(); - writer.writeUInt16(mode != -1 ? mode : 0xFFFF); - writer.writeUInt16(0); + writer.write(mode); + writer.write(0); break; case TYPE: int type = ((LexerTypeAction)action).getType(); - writer.writeUInt16(type != -1 ? type : 0xFFFF); - writer.writeUInt16(0); + writer.write(type); + writer.write(0); break; default: @@ -321,9 +294,9 @@ else if ( s instanceof BlockStartState ) { return data; } - private static void serializeSets(ATNDataWriter writer, Collection sets, UnicodeSerializeMode mode) { + private static void serializeSets(ATNDataWriter writer, Set sets) { int nSets = sets.size(); - writer.writeUInt16(nSets); + writer.write(nSets); for (IntervalSet set : sets) { boolean containsEof = set.contains(Token.EOF); @@ -331,9 +304,9 @@ private static void serializeSets(ATNDataWriter writer, Collection if (containsEof && set.getIntervals().get(0).b == Token.EOF) { size--; } - writer.writeUInt16(size); + writer.write(size); - writer.writeUInt16(containsEof ? 1 : 0); + writer.write(containsEof ? 1 : 0); for (Interval I : set.getIntervals()) { int firstValue; if (I.a == Token.EOF) { @@ -348,13 +321,8 @@ private static void serializeSets(ATNDataWriter writer, Collection firstValue = I.a; } - if (mode == UnicodeSerializeMode.UNICODE_BMP) { - writer.writeUInt16(firstValue); - writer.writeUInt16(I.b); - } else { - writer.writeUInt32(firstValue); - writer.writeUInt32(I.b); - } + writer.write(firstValue); + writer.write(I.b); } } } @@ -368,25 +336,22 @@ public String decode(char[] data) { throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason)); } - reader.readUInt16(); // skip grammarType - int maxType = reader.readUInt16(); + reader.read(); // skip grammarType + int maxType = reader.read(); buf.append("max type ").append(maxType).append("\n"); - int nstates = reader.readUInt16(); + int nstates = reader.read(); for (int i=0; i").append(trg) .append(" ").append(Transition.serializationNames.get(ttype)) .append(" ").append(arg1).append(",").append(arg2).append(",").append(arg3) .append("\n"); } - int ndecisions = reader.readUInt16(); + int ndecisions = reader.read(); for (int i=0; i sets) - { - return new ATNDeserializer().edgeFactory(atn, type, src, trg, arg1, arg2, arg3, sets); - } - /** * @deprecated Use {@link ATNDeserializer#stateFactory} instead. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java index 01301672fd..9320f8bb58 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java @@ -6,9 +6,8 @@ package org.antlr.v4.test.tool; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; @@ -18,6 +17,7 @@ import java.util.Arrays; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class TestATNDeserialization extends BaseJavaToolTest { @Before @@ -160,6 +160,52 @@ public void testSetUp() throws Exception { checkDeserializationIsStable(lg); } + @Test public void testATNDataWriterReaderCompact() { + IntegerList integerList = new IntegerList(); + ATNDataWriter writer = new ATNDataWriter(integerList, "Java"); + assertEquals(1, writer.write(0)); + assertEquals(1, writer.write(-1)); + assertEquals(1, writer.write(42)); + assertEquals(2, writer.write(1 << 14)); + assertEquals(2, writer.write(0xFFFF)); + assertEquals(3, writer.write(Integer.MAX_VALUE)); + assertEquals(3, writer.write(Integer.MIN_VALUE)); + assertEquals(13, integerList.size()); + + char[] charArray = Utils.toCharArray(integerList); + ATNDataReader reader = new ATNDataReader(charArray); + assertEquals(0, reader.read()); + assertEquals(-1, reader.read()); + assertEquals(42, reader.read()); + assertEquals(1 << 14, reader.read()); + assertEquals(0xFFFF, reader.read()); + assertEquals(Integer.MAX_VALUE, reader.read()); + assertEquals(Integer.MIN_VALUE, reader.read()); + } + + @Test public void testATNDataWriterReaderRaw() { + IntegerList integerList = new IntegerList(); + ATNDataWriter writer = new ATNDataWriter(integerList, "Java"); + writer.writeInt32(0); + writer.writeInt32(-1); + writer.writeInt32(42); + writer.writeInt32(1 << 14); + writer.writeInt32(0xFFFF); + writer.writeInt32(Integer.MAX_VALUE); + writer.writeInt32(Integer.MIN_VALUE); + assertEquals(7 * 2, integerList.size()); + + char[] charArray = Utils.toCharArray(integerList); + ATNDataReader reader = new ATNDataReader(charArray); + assertEquals(0, reader.readInt32()); + assertEquals(-1, reader.readInt32()); + assertEquals(42, reader.readInt32()); + assertEquals(1 << 14, reader.readInt32()); + assertEquals(0xFFFF, reader.readInt32()); + assertEquals(Integer.MAX_VALUE, reader.readInt32()); + assertEquals(Integer.MIN_VALUE, reader.readInt32()); + } + protected void checkDeserializationIsStable(Grammar g) { ATN atn = createATN(g, false); char[] data = Utils.toCharArray(ATNSerializer.getSerialized(atn, "Java")); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java index e962e6131e..7a79c83010 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java @@ -921,12 +921,12 @@ public void testSetUp() throws Exception { "5:BASIC 0\n" + "rule 0:1 1\n" + "mode 0:0\n" + - "0:'a'..'b'\n" + - "1:'e'..'e', 'p'..'t'\n" + + "0:'e'..'e', 'p'..'t'\n" + + "1:'a'..'b'\n" + "0->1 EPSILON 0,0,0\n" + "1->3 EPSILON 0,0,0\n" + - "3->4 NOT_SET 0,0,0\n" + - "4->5 NOT_SET 1,0,0\n" + + "3->4 NOT_SET 1,0,0\n" + + "4->5 NOT_SET 0,0,0\n" + "5->2 EPSILON 0,0,0\n" + "0:0\n"; ATN atn = createATN(lg, true);