Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ATN serialized data: remove shifting by 2, remove UUID #3516

Merged
merged 3 commits into from
Feb 6, 2022
Merged

ATN serialized data: remove shifting by 2, remove UUID #3516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
14649e7
ATN serialized data: remove shifting by 2, remove UUID; fix #3515
KvanTTT Jan 30, 2022
8333d1a
Disable native runtime tests (see #3521)
KvanTTT Jan 31, 2022
60f92b8
Implement Java-specific ATN data optimization (+-2 shift)
KvanTTT Feb 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 14 additions & 17 deletions .circleci/scripts/run-tests-python2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,18 @@ set -euo pipefail

python2 --version

pushd runtime/Python2/tests
echo "running native tests..."
python2 run.py
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd
# TODO: https://github.com/antlr/antlr4/issues/3521
#
# pushd runtime/Python2/tests
# echo "running native tests..."
# python2 run.py
# rc=$?
# if [ $rc != 0 ]; then
# echo "failed running native tests"
# fi
# popd

if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=python2.** test
rc=$?
popd
fi

# return $rc
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=python2.** test
popd
31 changes: 14 additions & 17 deletions .circleci/scripts/run-tests-python3.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,18 @@ set -euo pipefail

python3 --version

pushd runtime/Python3/tests
echo "running native tests..."
python3 run.py
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd

if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=python3.** test
rc=$?
popd
fi
# TODO: https://github.com/antlr/antlr4/issues/3521
#
# pushd runtime/Python3/tests
# echo "running native tests..."
# python3 run.py
# rc=$?
# if [ $rc != 0 ]; then
# echo "failed running native tests"
# fi
# popd

# return $rc
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=python3.** test
popd
29 changes: 14 additions & 15 deletions .circleci/scripts/run-tests-swift.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,18 @@

set -euo pipefail

pushd runtime/Swift
echo "running native tests..."
./boot.py --test
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd

# TODO: https://github.com/antlr/antlr4/issues/3521
#
# pushd runtime/Swift
# echo "running native tests..."
# ./boot.py --test
# rc=$?
# if [ $rc != 0 ]; then
# echo "failed running native tests"
# fi
# popd

if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
popd
fi
pushd runtime-testsuite
echo "running maven tests..."
mvn -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
popd
23 changes: 12 additions & 11 deletions .github/scripts-macosx/run-tests-swift.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,19 @@ swift --version
swift build --version

# run swift tests
pushd runtime/Swift
./boot.py --test
rc=$?
popd

if [ $rc == 0 ]; then
# run java tests
cd runtime-testsuite/
# TODO: https://github.com/antlr/antlr4/issues/3521
# pushd runtime/Swift
# ./boot.py --test
# rc=$?
# popd

# run java tests
cd runtime-testsuite/
# mvn -e -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
# I don't know swift enough to make it parallel. revert to single threaded
mvn -e -Dtest=swift.** test
rc=$?
cat target/surefire-reports/*.dumpstream || true
fi
mvn -e -Dtest=swift.** test
rc=$?
cat target/surefire-reports/*.dumpstream || true

exit $rc
2 changes: 1 addition & 1 deletion runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ protected ATN createATN(Grammar g, boolean useSerializer) {

ATN atn = g.atn;
if ( useSerializer ) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
char[] serialized = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());
return new ATNDeserializer().deserialize(serialized);
}

Expand Down
2 changes: 1 addition & 1 deletion runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestInterpreterDataReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void testParseFile() throws IOException, NoSuchFieldException, IllegalAcc
Assert.assertNull(channels);
Assert.assertNull(modes);

char[] atnChars = ATNSerializer.getSerializedAsChars(atn);
char[] atnChars = ATNSerializer.getSerializedAsChars(atn, g.getLanguage());
Assert.assertEquals(ATNDeserializer.SERIALIZED_VERSION, atnChars[0]);
}

Expand Down
140 changes: 5 additions & 135 deletions runtime/CSharp/src/Atn/ATNDeserializer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using Antlr4.Runtime.Atn;
using Antlr4.Runtime.Dfa;
using Antlr4.Runtime.Misc;
using Antlr4.Runtime.Sharpen;
Expand All @@ -16,53 +14,11 @@ namespace Antlr4.Runtime.Atn
/// <author>Sam Harwell</author>
public class ATNDeserializer
{
public static readonly int SerializedVersion = 3;

/// <summary>This is the earliest supported serialized UUID.</summary>
/// <remarks>This is the earliest supported serialized UUID.</remarks>
private static readonly Guid BaseSerializedUuid;

/// <summary>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </summary>
/// <remarks>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </remarks>
private static readonly Guid AddedUnicodeSmp;

/// <summary>
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
/// </summary>
/// <remarks>
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
/// </remarks>
private static readonly IList<Guid> SupportedUuids;

/// <summary>This is the current serialized UUID.</summary>
/// <remarks>This is the current serialized UUID.</remarks>
public static readonly Guid SerializedUuid;

static ATNDeserializer()
{
BaseSerializedUuid = new Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
AddedUnicodeSmp = new Guid("59627784-3BE5-417A-B9EB-8131A7286089");
SupportedUuids = new List<Guid>();
SupportedUuids.Add(BaseSerializedUuid);
SupportedUuids.Add(AddedUnicodeSmp);
SerializedUuid = AddedUnicodeSmp;
}
public static readonly int SerializedVersion = 4;

[NotNull]
private readonly ATNDeserializationOptions deserializationOptions;

private Guid uuid;

public ATNDeserializer()
: this(ATNDeserializationOptions.Default)
{
Expand All @@ -77,66 +33,21 @@ public ATNDeserializer(ATNDeserializationOptions deserializationOptions)
this.deserializationOptions = deserializationOptions;
}

/// <summary>
/// Determines if a particular serialized representation of an ATN supports
/// a particular feature, identified by the
/// <see cref="Guid"/>
/// used for serializing
/// the ATN at the time the feature was first introduced.
/// </summary>
/// <param name="feature">
/// The
/// <see cref="Guid"/>
/// marking the first time the feature was
/// supported in the serialized ATN.
/// </param>
/// <param name="actualUuid">
/// The
/// <see cref="Guid"/>
/// of the actual serialized ATN which is
/// currently being deserialized.
/// </param>
/// <returns>
///
/// <see langword="true"/>
/// if the
/// <paramref name="actualUuid"/>
/// value represents a
/// serialized ATN at or after the feature identified by
/// <paramref name="feature"/>
/// was
/// introduced; otherwise,
/// <see langword="false"/>
/// .
/// </returns>
protected internal virtual bool IsFeatureSupported(Guid feature, Guid actualUuid)
{
int featureIndex = SupportedUuids.IndexOf(feature);
if (featureIndex < 0)
{
return false;
}
return SupportedUuids.IndexOf(actualUuid) >= featureIndex;
}

char[] data;
int p;

public virtual ATN Deserialize(char[] data)
{
Reset (data);
this.data = data;
CheckVersion ();
CheckUUID ();
ATN atn = ReadATN ();
ReadStates (atn);
ReadRules (atn);
ReadModes (atn);
IList<IntervalSet> sets = new List<IntervalSet>();
ReadSets (atn, sets, this.ReadInt);
if (IsFeatureSupported(AddedUnicodeSmp, uuid)) {
ReadSets (atn, sets, this.ReadInt32);
}
ReadEdges (atn, sets);
ReadSets (atn, sets, ReadInt);
ReadSets (atn, sets, ReadInt32);
ReadEdges (atn, sets);
ReadDecisions (atn);
ReadLexerActions (atn);
MarkPrecedenceDecisions(atn);
Expand Down Expand Up @@ -546,16 +457,6 @@ protected internal virtual ATN ReadATN()
return new ATN(grammarType, maxTokenType);
}

protected internal virtual void CheckUUID()
{
uuid = ReadUUID();
if (!SupportedUuids.Contains(uuid))
{
string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID {0} (expected {1} or a legacy UUID).", uuid, SerializedUuid);
throw new NotSupportedException(reason);
}
}

protected internal virtual void CheckVersion()
{
int version = ReadInt();
Expand All @@ -566,18 +467,6 @@ protected internal virtual void CheckVersion()
}
}

protected internal virtual void Reset(char[] data)
{
this.data = new char[data.Length];
// don't adjust the first value since that's the version number
this.data[0] = data[0];
for (int i = 1; i < data.Length; i++)
{
this.data[i] = (char)(data[i] - 2);
}
this.p = 0;
}

/// <summary>
/// Analyze the
/// <see cref="StarLoopEntryState"/>
Expand Down Expand Up @@ -1083,25 +972,6 @@ protected internal int ReadInt32()
return (int)data[p++] | ((int)data[p++] << 16);
}

protected internal long ReadLong()
{
long lowOrder = ReadInt32() & unchecked((long)(0x00000000FFFFFFFFL));
return lowOrder | ((long)ReadInt32() << 32);
}

protected internal Guid ReadUUID()
{
byte[] d = BitConverter.GetBytes (ReadLong ());
if(BitConverter.IsLittleEndian)
{
Array.Reverse(d);
}
short c = (short)ReadInt();
short b = (short)ReadInt();
int a = ReadInt32();
return new Guid(a, b, c, d);
}

[return: NotNull]
protected internal virtual Transition EdgeFactory(ATN atn, TransitionType type, int src, int trg, int arg1, int arg2, int arg3, IList<IntervalSet> sets)
{
Expand Down
Loading