Skip to content

Commit

Permalink
Stricter pre-processor directive recognition (#2)
Browse files Browse the repository at this point in the history 
Make pre-processor directive recognition stricter

Recognise a pre-processor directive either at start of a line or when
preceded only by horizontal whitespace.
  • Loading branch information
@atifaziz
atifaziz authored Mar 28, 2019
1 parent 934e257 commit 1e8a189
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 17 deletions.
60 changes: 43 additions & 17 deletions src/CSharpMinifier/Scanner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public static IEnumerable<Token> Scan(string source)

enum State
{
NewLine,
LeadingWhiteSpace,
Text,
WhiteSpace,
Cr,
Expand Down Expand Up @@ -61,14 +63,13 @@ enum State

static IEnumerable<Token> ScanImpl(string source)
{
var state = State.Text;
var state = State.NewLine;
var si = 0;
var pos = (Line: 1, Col: 0);
var spos = (Line: 1, Col: 1);
var ppdtwssi = -1;
var ppdtwscol = 0;
int i;
var lastTokenKind = (TokenKind?)null;
var interpolated = new Stack<(bool Verbatim, int Parens)>();

bool Interpolated() => interpolated.Count > 0;
Expand All @@ -87,12 +88,9 @@ T TransitReturn<T>(State newState, int offset, T token)
return token;
}

Token CreateToken(TokenKind kind, int offset = 0)
{
lastTokenKind = kind;
return new Token(kind, new Position(si, spos.Line, spos.Col),
new Position(i + offset, pos.Line, pos.Col + offset));
}
Token CreateToken(TokenKind kind, int offset = 0) =>
new Token(kind, new Position(si, spos.Line, spos.Col),
new Position(i + offset, pos.Line, pos.Col + offset));

Token? TextTransit(State newState, int offset = 0) =>
TransitReturn(newState, offset,
Expand Down Expand Up @@ -132,6 +130,39 @@ Exception SyntaxError(string message) =>
restart:
switch (state)
{
case State.NewLine:
{
switch (ch)
{
case ' ':
case '\t':
state = State.LeadingWhiteSpace;
break;
case '#':
state = State.PreprocessorDirective;
break;
default:
state = State.Text;
goto restart;
}
break;
}
case State.LeadingWhiteSpace:
{
switch (ch)
{
case ' ':
case '\t':
break;
case '#':
yield return Transit(TokenKind.WhiteSpace, State.PreprocessorDirective);
break;
default:
yield return Transit(TokenKind.WhiteSpace, State.Text);
goto restart;
}
break;
}
case State.Text:
{
switch (ch)
Expand Down Expand Up @@ -175,11 +206,6 @@ Exception SyntaxError(string message) =>
throw SyntaxError("Parentheses mismatch in interpolated string expression.");
break;
}
case '#' when lastTokenKind is null // BOF
|| lastTokenKind is TokenKind k
&& (k == TokenKind.WhiteSpace || k == TokenKind.NewLine):
state = State.PreprocessorDirective;
goto restart;
case ' ':
case '\t':
{
Expand All @@ -198,7 +224,7 @@ Exception SyntaxError(string message) =>
if (TextTransit(State.Text) is Token text)
yield return text;
pos = (pos.Line + 1, 0);
yield return Transit(TokenKind.NewLine, State.Text, 1);
yield return Transit(TokenKind.NewLine, State.NewLine, 1);
break;
}
}
Expand Down Expand Up @@ -227,11 +253,11 @@ Exception SyntaxError(string message) =>
break;
case '\n':
pos = (pos.Line + 1, 0);
yield return Transit(TokenKind.NewLine, State.Text, 1);
yield return Transit(TokenKind.NewLine, State.NewLine, 1);
break;
default:
pos = (pos.Line + 1, 1);
yield return Transit(TokenKind.NewLine, State.Text);
yield return Transit(TokenKind.NewLine, State.NewLine);
goto restart;
}
break;
Expand Down Expand Up @@ -612,7 +638,7 @@ Exception SyntaxError(string message) =>
{
var token
= state == State.SingleLineComment ? TokenKind.SingleLineComment
: state == State.WhiteSpace ? TokenKind.WhiteSpace
: state == State.WhiteSpace || state == State.LeadingWhiteSpace ? TokenKind.WhiteSpace
: state == State.Cr ? TokenKind.NewLine
: state == State.PreprocessorDirective || state == State.PreprocessorDirectiveSlash ? TokenKind.PreprocessorDirective
: state == State.VerbatimStringQuote ? TokenKind.VerbatimStringLiteral
Expand Down
28 changes: 28 additions & 0 deletions tests/ScannerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,17 @@ public void SyntaxError(string source)
@"NewLine 1 1 =1 ""\r""",
@"Text 2 0 2 ""42""")]

// A pre-processing directive always occupies a separate line of
// source code and always begins with a # character and a
// pre-processing directive name.

[TestCase("foo #bar baz", // not technically valid C#
@"Text 3 0 3 ""foo""",
@"WhiteSpace 1 0 1 "" """,
@"Text 4 0 4 ""#bar""",
@"WhiteSpace 1 0 1 "" """,
@"Text 3 0 3 ""baz""")]

[TestCase("#line 42" , @"PreprocessorDirective 8 0 8 ""#line 42""")]
[TestCase("#line 42 / / comment", @"PreprocessorDirective 20 0 20 ""#line 42 / / comment""")]
[TestCase("#line 42/ /comment" , @"PreprocessorDirective 18 0 18 ""#line 42/ /comment""")]
Expand Down Expand Up @@ -207,6 +218,23 @@ public void SyntaxError(string source)
@"PreprocessorDirective 11 0 11 ""#error 42 /""",
@"NewLine 1 1 -11 ""\n""")]

// White space may occur before
// the # character and between the # character and the directive
// name.

[TestCase("# error 42",
@"PreprocessorDirective 10 0 10 ""# error 42""")]

// Delimited comments (the /* */ style of comments) are not permitted
// on source lines containing pre-processing directives.

[TestCase("/* foo */ #bar /* baz */",
@"MultiLineComment 9 0 9 ""/* foo */""",
@"WhiteSpace 1 0 1 "" """,
@"Text 4 0 4 ""#bar""",
@"WhiteSpace 1 0 1 "" """,
@"MultiLineComment 9 0 9 ""/* baz */""")]

[TestCase("@\"\"",
@"VerbatimString 3 0 3 ""@\""\""""")]

Expand Down

0 comments on commit 1e8a189

Please sign in to comment.