Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lexer: avoid unnecessary allocations on compile-time arrays #449

Merged
merged 2 commits into from
May 4, 2023
Merged

lexer: avoid unnecessary allocations on compile-time arrays #449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6883639
lexer: avoid unnecessary allocations on compile-time arrays
ljmf00 Sep 3, 2021
27ae689
hacky upgrade GDC
WebFreak001 May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ jobs:
- name: Install GDC
if: ${{ matrix.compiler == 'gdc-latest' }}
run: |
sudo apt-get install gdc gdmd -y
sudo apt-get install gdc-12 gdmd -y
# hack:
sudo rm /usr/bin/gdc
sudo ln -s /usr/bin/gdc-12 /usr/bin/gdc
gdc --version

# Execute all other tests (parsing, XPath, ...)
Expand Down
14 changes: 7 additions & 7 deletions src/dparse/lexer.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import core.cpuid : sse42;
public import dparse.trivia;

/// Operators
private enum operators = [
private immutable operators = [
",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
"!>", "!>=", "$", "%", "%=", "&", "&&", "&=", "(", ")", "*", "*=", "+", "++",
"+=", "-", "--", "-=", ":", ";", "<", "<<", "<<=", "<=", "<>", "<>=", "=",
Expand All @@ -21,7 +21,7 @@ private enum operators = [
];

/// Kewords
private enum keywords = [
private immutable keywords = [
"abstract", "alias", "align", "asm", "assert", "auto", "bool",
"break", "byte", "case", "cast", "catch", "cdouble", "cent", "cfloat",
"char", "class", "const", "continue", "creal", "dchar", "debug", "default",
Expand All @@ -42,15 +42,15 @@ private enum keywords = [
];

/// Other tokens
private enum dynamicTokens = [
private immutable dynamicTokens = [
"specialTokenSequence", "comment", "identifier", "scriptLine",
"whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral",
"ifloatLiteral", "intLiteral", "longLiteral", "realLiteral",
"irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral",
"dstringLiteral", "stringLiteral", "wstringLiteral"
];

private enum pseudoTokenHandlers = [
private immutable pseudoTokenHandlers = [
"\"", "lexStringLiteral",
"`", "lexWysiwygString",
"//", "lexSlashSlashComment",
Expand Down Expand Up @@ -171,8 +171,8 @@ mixin template TokenTriviaFields()

// mixin in from dparse.lexer to make error messages more managable size as the
// entire string is dumped when there is a type mismatch.
private enum extraFields = "import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;";
private enum extraFieldsBare = "
private immutable extraFields = "import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;";
private immutable extraFieldsBare = "
import dparse.lexer : Token;

this(Token token) pure nothrow @safe @nogc {
Expand Down Expand Up @@ -205,7 +205,7 @@ public enum WhitespaceBehavior : ubyte
skip = 0b0000_0001,
}

private enum stringBehaviorNotWorking = "Automatic string parsing is not "
private immutable stringBehaviorNotWorking = "Automatic string parsing is not "
~ "supported and was previously not working. To unescape strings use the "
~ "`dparse.strings : unescapeString` function on the token texts instead.";

Expand Down
14 changes: 7 additions & 7 deletions src/std/experimental/lexer.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ mixin template Lexer(Token, alias defaultTokenFunction,
return i;
}

private static char[] getBeginningChars(string[] allTokens)
private static char[] getBeginningChars(immutable string[] allTokens)
{
char[] beginningChars;
for (size_t i = 0; i < allTokens.length; i++)
Expand All @@ -469,17 +469,17 @@ mixin template Lexer(Token, alias defaultTokenFunction,
import std.algorithm : sort;
import std.range : stride;

string[] pseudoTokens = array(tokenHandlers.stride(2));
string[] allTokens = array(sort(staticTokens ~ possibleDefaultTokens ~ pseudoTokens).uniq());
immutable string[] pseudoTokens = array(tokenHandlers.stride(2));
immutable string[] allTokens = array(sort((staticTokens ~ possibleDefaultTokens ~ pseudoTokens).dup).uniq());
// Array consisting of a sorted list of the first characters of the
// tokens.
char[] beginningChars = getBeginningChars(allTokens);
size_t i = calcSplitCount(beginningChars.length, 8);
return generateStatementsStep(allTokens, pseudoTokens, beginningChars, i);
}

private static string generateStatementsStep(string[] allTokens,
string[] pseudoTokens, char[] chars, size_t i, string indent = "")
private static string generateStatementsStep(immutable string[] allTokens,
immutable string[] pseudoTokens, char[] chars, size_t i, string indent = "")
{
import std.string : format;
string code;
Expand Down Expand Up @@ -519,12 +519,12 @@ mixin template Lexer(Token, alias defaultTokenFunction,
return code;
}

private static string printCase(string[] tokens, string[] pseudoTokens, string indent)
private static string printCase(immutable string[] tokens, immutable string[] pseudoTokens, string indent)
{
import std.array : array;
import std.algorithm : countUntil;
import std.conv : text;
string[] sortedTokens = array(sort!"a.length > b.length"(tokens));
immutable string[] sortedTokens = array(sort!"a.length > b.length"(tokens.dup));

if (tokens.length == 1 && tokens[0].length == 1)
{
Expand Down