Skip to content

Commit

Permalink
ATN serialized data: remove shifting by 2, remove UUID; fix #3515
Browse files Browse the repository at this point in the history
  • Loading branch information
@KvanTTT
KvanTTT committed Jan 30, 2022
1 parent 9e04c9f commit 88e10ac
Show file tree
Hide file tree
Showing 17 changed files with 149 additions and 1,374 deletions.
2 changes: 1 addition & 1 deletion runtime-testsuite/resources/org/antlr/v4/test/runtime/InterpDataReaderTest.interp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,4 @@ mode names:
DEFAULT_MODE

atn:
[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 13, 61, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 6, 9, 41, 10, 9, 13, 9, 14, 9, 42, 3, 10, 6, 10, 46, 10, 10, 13, 10, 14, 10, 47, 3, 11, 5, 11, 51, 10, 11, 3, 11, 3, 11, 3, 12, 6, 12, 56, 10, 12, 13, 12, 14, 12, 57, 3, 12, 3, 12, 2, 2, 13, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 3, 2, 5, 4, 2, 67, 92, 99, 124, 3, 2, 50, 59, 4, 2, 11, 11, 34, 34, 2, 64, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 3, 25, 3, 2, 2, 2, 5, 27, 3, 2, 2, 2, 7, 29, 3, 2, 2, 2, 9, 31, 3, 2, 2, 2, 11, 33, 3, 2, 2, 2, 13, 35, 3, 2, 2, 2, 15, 37, 3, 2, 2, 2, 17, 40, 3, 2, 2, 2, 19, 45, 3, 2, 2, 2, 21, 50, 3, 2, 2, 2, 23, 55, 3, 2, 2, 2, 25, 26, 7, 63, 2, 2, 26, 4, 3, 2, 2, 2, 27, 28, 7, 44, 2, 2, 28, 6, 3, 2, 2, 2, 29, 30, 7, 49, 2, 2, 30, 8, 3, 2, 2, 2, 31, 32, 7, 45, 2, 2, 32, 10, 3, 2, 2, 2, 33, 34, 7, 47, 2, 2, 34, 12, 3, 2, 2, 2, 35, 36, 7, 42, 2, 2, 36, 14, 3, 2, 2, 2, 37, 38, 7, 43, 2, 2, 38, 16, 3, 2, 2, 2, 39, 41, 9, 2, 2, 2, 40, 39, 3, 2, 2, 2, 41, 42, 3, 2, 2, 2, 42, 40, 3, 2, 2, 2, 42, 43, 3, 2, 2, 2, 43, 18, 3, 2, 2, 2, 44, 46, 9, 3, 2, 2, 45, 44, 3, 2, 2, 2, 46, 47, 3, 2, 2, 2, 47, 45, 3, 2, 2, 2, 47, 48, 3, 2, 2, 2, 48, 20, 3, 2, 2, 2, 49, 51, 7, 15, 2, 2, 50, 49, 3, 2, 2, 2, 50, 51, 3, 2, 2, 2, 51, 52, 3, 2, 2, 2, 52, 53, 7, 12, 2, 2, 53, 22, 3, 2, 2, 2, 54, 56, 9, 4, 2, 2, 55, 54, 3, 2, 2, 2, 56, 57, 3, 2, 2, 2, 57, 55, 3, 2, 2, 2, 57, 58, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 60, 8, 12, 2, 2, 60, 24, 3, 2, 2, 2, 7, 2, 42, 47, 50, 57, 3, 8, 2, 2]
[4, 0, 11, 59, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 6, 1, 6, 1, 7, 4, 7, 39, 8, 7, 11, 7, 12, 7, 40, 1, 8, 4, 8, 44, 8, 8, 11, 8, 12, 8, 45, 1, 9, 3, 9, 49, 8, 9, 1, 9, 1, 9, 1, 10, 4, 10, 54, 8, 10, 11, 10, 12, 10, 55, 1, 10, 1, 10, 0, 0, 11, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 1, 0, 3, 2, 0, 65, 90, 97, 122, 1, 0, 48, 57, 2, 0, 9, 9, 32, 32, 0, 62, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 1, 23, 1, 0, 0, 0, 3, 25, 1, 0, 0, 0, 5, 27, 1, 0, 0, 0, 7, 29, 1, 0, 0, 0, 9, 31, 1, 0, 0, 0, 11, 33, 1, 0, 0, 0, 13, 35, 1, 0, 0, 0, 15, 38, 1, 0, 0, 0, 17, 43, 1, 0, 0, 0, 19, 48, 1, 0, 0, 0, 21, 53, 1, 0, 0, 0, 23, 24, 5, 61, 0, 0, 24, 2, 1, 0, 0, 0, 25, 26, 5, 42, 0, 0, 26, 4, 1, 0, 0, 0, 27, 28, 5, 47, 0, 0, 28, 6, 1, 0, 0, 0, 29, 30, 5, 43, 0, 0, 30, 8, 1, 0, 0, 0, 31, 32, 5, 45, 0, 0, 32, 10, 1, 0, 0, 0, 33, 34, 5, 40, 0, 0, 34, 12, 1, 0, 0, 0, 35, 36, 5, 41, 0, 0, 36, 14, 1, 0, 0, 0, 37, 39, 7, 0, 0, 0, 38, 37, 1, 0, 0, 0, 39, 40, 1, 0, 0, 0, 40, 38, 1, 0, 0, 0, 40, 41, 1, 0, 0, 0, 41, 16, 1, 0, 0, 0, 42, 44, 7, 1, 0, 0, 43, 42, 1, 0, 0, 0, 44, 45, 1, 0, 0, 0, 45, 43, 1, 0, 0, 0, 45, 46, 1, 0, 0, 0, 46, 18, 1, 0, 0, 0, 47, 49, 5, 13, 0, 0, 48, 47, 1, 0, 0, 0, 48, 49, 1, 0, 0, 0, 49, 50, 1, 0, 0, 0, 50, 51, 5, 10, 0, 0, 51, 20, 1, 0, 0, 0, 52, 54, 7, 2, 0, 0, 53, 52, 1, 0, 0, 0, 54, 55, 1, 0, 0, 0, 55, 53, 1, 0, 0, 0, 55, 56, 1, 0, 0, 0, 56, 57, 1, 0, 0, 0, 57, 58, 6, 10, 0, 0, 58, 22, 1, 0, 0, 0, 5, 0, 40, 45, 48, 55, 1, 6, 0, 0]
2 changes: 1 addition & 1 deletion runtime-testsuite/resources/org/antlr/v4/test/runtime/InterpDataReaderTest2.interp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,4 @@ mode names:
DEFAULT_MODE

atn:
[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 13, 61, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 6, 9, 41, 10, 9, 13, 9, 14, 9, 42, 3, 10, 6, 10, 46, 10, 10, 13, 10, 14, 10, 47, 3, 11, 5, 11, 51, 10, 11, 3, 11, 3, 11, 3, 12, 6, 12, 56, 10, 12, 13, 12, 14, 12, 57, 3, 12, 3, 12, 2, 2, 13, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 3, 2, 5, 4, 2, 67, 92, 99, 124, 3, 2, 50, 59, 4, 2, 11, 11, 34, 34, 2, 64, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 3, 25, 3, 2, 2, 2, 5, 27, 3, 2, 2, 2, 7, 29, 3, 2, 2, 2, 9, 31, 3, 2, 2, 2, 11, 33, 3, 2, 2, 2, 13, 35, 3, 2, 2, 2, 15, 37, 3, 2, 2, 2, 17, 40, 3, 2, 2, 2, 19, 45, 3, 2, 2, 2, 21, 50, 3, 2, 2, 2, 23, 55, 3, 2, 2, 2, 25, 26, 7, 63, 2, 2, 26, 4, 3, 2, 2, 2, 27, 28, 7, 44, 2, 2, 28, 6, 3, 2, 2, 2, 29, 30, 7, 49, 2, 2, 30, 8, 3, 2, 2, 2, 31, 32, 7, 45, 2, 2, 32, 10, 3, 2, 2, 2, 33, 34, 7, 47, 2, 2, 34, 12, 3, 2, 2, 2, 35, 36, 7, 42, 2, 2, 36, 14, 3, 2, 2, 2, 37, 38, 7, 43, 2, 2, 38, 16, 3, 2, 2, 2, 39, 41, 9, 2, 2, 2, 40, 39, 3, 2, 2, 2, 41, 42, 3, 2, 2, 2, 42, 40, 3, 2, 2, 2, 42, 43, 3, 2, 2, 2, 43, 18, 3, 2, 2, 2, 44, 46, 9, 3, 2, 2, 45, 44, 3, 2, 2, 2, 46, 47, 3, 2, 2, 2, 47, 45, 3, 2, 2, 2, 47, 48, 3, 2, 2, 2, 48, 20, 3, 2, 2, 2, 49, 51, 7, 15, 2, 2, 50, 49, 3, 2, 2, 2, 50, 51, 3, 2, 2, 2, 51, 52, 3, 2, 2, 2, 52, 53, 7, 12, 2, 2, 53, 22, 3, 2, 2, 2, 54, 56, 9, 4, 2, 2, 55, 54, 3, 2, 2, 2, 56, 57, 3, 2, 2, 2, 57, 55, 3, 2, 2, 2, 57, 58, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 60, 8, 12, 2, 2, 60, 24, 3, 2, 2, 2, 7, 2, 42, 47, 50, 57, 3, 8, 2, 2]
[4, 0, 11, 59, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 6, 1, 6, 1, 7, 4, 7, 39, 8, 7, 11, 7, 12, 7, 40, 1, 8, 4, 8, 44, 8, 8, 11, 8, 12, 8, 45, 1, 9, 3, 9, 49, 8, 9, 1, 9, 1, 9, 1, 10, 4, 10, 54, 8, 10, 11, 10, 12, 10, 55, 1, 10, 1, 10, 0, 0, 11, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 1, 0, 3, 2, 0, 65, 90, 97, 122, 1, 0, 48, 57, 2, 0, 9, 9, 32, 32, 0, 62, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 1, 23, 1, 0, 0, 0, 3, 25, 1, 0, 0, 0, 5, 27, 1, 0, 0, 0, 7, 29, 1, 0, 0, 0, 9, 31, 1, 0, 0, 0, 11, 33, 1, 0, 0, 0, 13, 35, 1, 0, 0, 0, 15, 38, 1, 0, 0, 0, 17, 43, 1, 0, 0, 0, 19, 48, 1, 0, 0, 0, 21, 53, 1, 0, 0, 0, 23, 24, 5, 61, 0, 0, 24, 2, 1, 0, 0, 0, 25, 26, 5, 42, 0, 0, 26, 4, 1, 0, 0, 0, 27, 28, 5, 47, 0, 0, 28, 6, 1, 0, 0, 0, 29, 30, 5, 43, 0, 0, 30, 8, 1, 0, 0, 0, 31, 32, 5, 45, 0, 0, 32, 10, 1, 0, 0, 0, 33, 34, 5, 40, 0, 0, 34, 12, 1, 0, 0, 0, 35, 36, 5, 41, 0, 0, 36, 14, 1, 0, 0, 0, 37, 39, 7, 0, 0, 0, 38, 37, 1, 0, 0, 0, 39, 40, 1, 0, 0, 0, 40, 38, 1, 0, 0, 0, 40, 41, 1, 0, 0, 0, 41, 16, 1, 0, 0, 0, 42, 44, 7, 1, 0, 0, 43, 42, 1, 0, 0, 0, 44, 45, 1, 0, 0, 0, 45, 43, 1, 0, 0, 0, 45, 46, 1, 0, 0, 0, 46, 18, 1, 0, 0, 0, 47, 49, 5, 13, 0, 0, 48, 47, 1, 0, 0, 0, 48, 49, 1, 0, 0, 0, 49, 50, 1, 0, 0, 0, 50, 51, 5, 10, 0, 0, 51, 20, 1, 0, 0, 0, 52, 54, 7, 2, 0, 0, 53, 52, 1, 0, 0, 0, 54, 55, 1, 0, 0, 0, 55, 53, 1, 0, 0, 0, 55, 56, 1, 0, 0, 0, 56, 57, 1, 0, 0, 0, 57, 58, 6, 10, 0, 0, 58, 22, 1, 0, 0, 0, 5, 0, 40, 45, 48, 55, 1, 6, 0, 0]
140 changes: 5 additions & 135 deletions runtime/CSharp/src/Atn/ATNDeserializer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using Antlr4.Runtime.Atn;
using Antlr4.Runtime.Dfa;
using Antlr4.Runtime.Misc;
using Antlr4.Runtime.Sharpen;
Expand All @@ -16,53 +14,11 @@ namespace Antlr4.Runtime.Atn
/// <author>Sam Harwell</author>
public class ATNDeserializer
{
public static readonly int SerializedVersion = 3;

/// <summary>This is the earliest supported serialized UUID.</summary>
/// <remarks>This is the earliest supported serialized UUID.</remarks>
private static readonly Guid BaseSerializedUuid;

/// <summary>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </summary>
/// <remarks>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </remarks>
private static readonly Guid AddedUnicodeSmp;

/// <summary>
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
/// </summary>
/// <remarks>
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
/// </remarks>
private static readonly IList<Guid> SupportedUuids;

/// <summary>This is the current serialized UUID.</summary>
/// <remarks>This is the current serialized UUID.</remarks>
public static readonly Guid SerializedUuid;

static ATNDeserializer()
{
BaseSerializedUuid = new Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
AddedUnicodeSmp = new Guid("59627784-3BE5-417A-B9EB-8131A7286089");
SupportedUuids = new List<Guid>();
SupportedUuids.Add(BaseSerializedUuid);
SupportedUuids.Add(AddedUnicodeSmp);
SerializedUuid = AddedUnicodeSmp;
}
public static readonly int SerializedVersion = 4;

[NotNull]
private readonly ATNDeserializationOptions deserializationOptions;

private Guid uuid;

public ATNDeserializer()
: this(ATNDeserializationOptions.Default)
{
Expand All @@ -77,66 +33,21 @@ public ATNDeserializer(ATNDeserializationOptions deserializationOptions)
this.deserializationOptions = deserializationOptions;
}

/// <summary>
/// Determines if a particular serialized representation of an ATN supports
/// a particular feature, identified by the
/// <see cref="Guid"/>
/// used for serializing
/// the ATN at the time the feature was first introduced.
/// </summary>
/// <param name="feature">
/// The
/// <see cref="Guid"/>
/// marking the first time the feature was
/// supported in the serialized ATN.
/// </param>
/// <param name="actualUuid">
/// The
/// <see cref="Guid"/>
/// of the actual serialized ATN which is
/// currently being deserialized.
/// </param>
/// <returns>
///
/// <see langword="true"/>
/// if the
/// <paramref name="actualUuid"/>
/// value represents a
/// serialized ATN at or after the feature identified by
/// <paramref name="feature"/>
/// was
/// introduced; otherwise,
/// <see langword="false"/>
/// .
/// </returns>
protected internal virtual bool IsFeatureSupported(Guid feature, Guid actualUuid)
{
int featureIndex = SupportedUuids.IndexOf(feature);
if (featureIndex < 0)
{
return false;
}
return SupportedUuids.IndexOf(actualUuid) >= featureIndex;
}

char[] data;
int p;

public virtual ATN Deserialize(char[] data)
{
Reset (data);
this.data = data;
CheckVersion ();
CheckUUID ();
ATN atn = ReadATN ();
ReadStates (atn);
ReadRules (atn);
ReadModes (atn);
IList<IntervalSet> sets = new List<IntervalSet>();
ReadSets (atn, sets, this.ReadInt);
if (IsFeatureSupported(AddedUnicodeSmp, uuid)) {
ReadSets (atn, sets, this.ReadInt32);
}
ReadEdges (atn, sets);
ReadSets (atn, sets, ReadInt);
ReadSets (atn, sets, ReadInt32);
ReadEdges (atn, sets);
ReadDecisions (atn);
ReadLexerActions (atn);
MarkPrecedenceDecisions(atn);
Expand Down Expand Up @@ -546,16 +457,6 @@ protected internal virtual ATN ReadATN()
return new ATN(grammarType, maxTokenType);
}

protected internal virtual void CheckUUID()
{
uuid = ReadUUID();
if (!SupportedUuids.Contains(uuid))
{
string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID {0} (expected {1} or a legacy UUID).", uuid, SerializedUuid);
throw new NotSupportedException(reason);
}
}

protected internal virtual void CheckVersion()
{
int version = ReadInt();
Expand All @@ -566,18 +467,6 @@ protected internal virtual void CheckVersion()
}
}

protected internal virtual void Reset(char[] data)
{
this.data = new char[data.Length];
// don't adjust the first value since that's the version number
this.data[0] = data[0];
for (int i = 1; i < data.Length; i++)
{
this.data[i] = (char)(data[i] - 2);
}
this.p = 0;
}

/// <summary>
/// Analyze the
/// <see cref="StarLoopEntryState"/>
Expand Down Expand Up @@ -1083,25 +972,6 @@ protected internal int ReadInt32()
return (int)data[p++] | ((int)data[p++] << 16);
}

protected internal long ReadLong()
{
long lowOrder = ReadInt32() & unchecked((long)(0x00000000FFFFFFFFL));
return lowOrder | ((long)ReadInt32() << 32);
}

protected internal Guid ReadUUID()
{
byte[] d = BitConverter.GetBytes (ReadLong ());
if(BitConverter.IsLittleEndian)
{
Array.Reverse(d);
}
short c = (short)ReadInt();
short b = (short)ReadInt();
int a = ReadInt32();
return new Guid(a, b, c, d);
}

[return: NotNull]
protected internal virtual Transition EdgeFactory(ATN atn, TransitionType type, int src, int trg, int arg1, int arg2, int arg3, IList<IntervalSet> sets)
{
Expand Down
Loading

0 comments on commit 88e10ac

Please sign in to comment.