Skip to content

Commit

Permalink
Partial commit - update bot token parser to store line number/column …
Browse files Browse the repository at this point in the history 
…info in each token for error reporting
  • Loading branch information
@ethanmoffat
ethanmoffat committed Feb 3, 2022
1 parent 6070889 commit a14260c
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 35 deletions.
9 changes: 8 additions & 1 deletion EOBot/Interpreter/BotToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,19 @@
public class BotToken
{
public BotTokenType TokenType { get; }

public string TokenValue { get; }

public BotToken(BotTokenType tokenType, string tokenValue)
public int LineNumber { get; }

public int Column { get; }

public BotToken(BotTokenType tokenType, string tokenValue, int line, int col)
{
TokenType = tokenType;
TokenValue = tokenValue;
LineNumber = line;
Column = col;
}

public override string ToString() => $"{TokenType}: {TokenValue}";
Expand Down
66 changes: 35 additions & 31 deletions EOBot/Interpreter/BotTokenParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public void Reset()
public BotToken GetNextToken()
{
if (_inputStream.EndOfStream)
return new BotToken(BotTokenType.EOF, string.Empty);
return Token(BotTokenType.EOF, string.Empty);

char inputChar;
do
Expand All @@ -54,7 +54,7 @@ public BotToken GetNextToken()
{
LineNumber++;
Column = 1;
return new BotToken(BotTokenType.NewLine, inputChar.ToString());
return Token(BotTokenType.NewLine, inputChar.ToString());
}

if (inputChar == '/' && !_inputStream.EndOfStream && (char)_inputStream.Peek() == '*')
Expand All @@ -79,7 +79,7 @@ public BotToken GetNextToken()

LineNumber++;
Column = 1;
return new BotToken(BotTokenType.NewLine, inputChar.ToString());
return Token(BotTokenType.NewLine, inputChar.ToString());
}
} while (!_inputStream.EndOfStream && char.IsWhiteSpace(inputChar));

Expand All @@ -93,62 +93,62 @@ public BotToken GetNextToken()
? BotTokenType.Keyword
: BotTokenType.Identifier;

return new BotToken(type, identifier);
return Token(type, identifier);
}
else if (char.IsDigit(inputChar))
{
var number = inputChar.ToString();
while (char.IsDigit((char)_inputStream.Peek()))
number += Read();
return new BotToken(BotTokenType.Literal, number);
return Token(BotTokenType.Literal, number);
}
else
{
switch(inputChar)
{
case '(': return new BotToken(BotTokenType.LParen, inputChar.ToString());
case ')': return new BotToken(BotTokenType.RParen, inputChar.ToString());
case '{': return new BotToken(BotTokenType.LBrace, inputChar.ToString());
case '}': return new BotToken(BotTokenType.RBrace, inputChar.ToString());
case '[': return new BotToken(BotTokenType.LBracket, inputChar.ToString());
case ']': return new BotToken(BotTokenType.RBracket, inputChar.ToString());
case ':': return new BotToken(BotTokenType.Colon, inputChar.ToString());
case ',': return new BotToken(BotTokenType.Comma, inputChar.ToString());
case '(': return Token(BotTokenType.LParen, inputChar.ToString());
case ')': return Token(BotTokenType.RParen, inputChar.ToString());
case '{': return Token(BotTokenType.LBrace, inputChar.ToString());
case '}': return Token(BotTokenType.RBrace, inputChar.ToString());
case '[': return Token(BotTokenType.LBracket, inputChar.ToString());
case ']': return Token(BotTokenType.RBracket, inputChar.ToString());
case ':': return Token(BotTokenType.Colon, inputChar.ToString());
case ',': return Token(BotTokenType.Comma, inputChar.ToString());
case '"':
{
var stringLiteral = string.Empty;
while ((char)_inputStream.Peek() != '"')
stringLiteral += Read();
Read();
return new BotToken(BotTokenType.Literal, stringLiteral);
return Token(BotTokenType.Literal, stringLiteral);
}
case '=':
{
switch((char)_inputStream.Peek())
{
case '=':
var nextChar = Read();
return new BotToken(BotTokenType.EqualOperator, inputChar.ToString() + nextChar);
return Token(BotTokenType.EqualOperator, inputChar.ToString() + nextChar);
default:
return new BotToken(BotTokenType.AssignOperator, inputChar.ToString());
return Token(BotTokenType.AssignOperator, inputChar.ToString());
}
}
case '!':
{
var nextChar = Read();
if (nextChar != '=')
return new BotToken(BotTokenType.Error, inputChar.ToString() + nextChar);
return new BotToken(BotTokenType.NotEqualOperator, inputChar.ToString() + nextChar);
return Token(BotTokenType.Error, inputChar.ToString() + nextChar);
return Token(BotTokenType.NotEqualOperator, inputChar.ToString() + nextChar);
}
case '>':
{
switch ((char)_inputStream.Peek())
{
case '=':
var nextChar = Read();
return new BotToken(BotTokenType.GreaterThanEqOperator, inputChar.ToString() + nextChar);
return Token(BotTokenType.GreaterThanEqOperator, inputChar.ToString() + nextChar);
default:
return new BotToken(BotTokenType.GreaterThanOperator, inputChar.ToString());
return Token(BotTokenType.GreaterThanOperator, inputChar.ToString());
}
}
case '<':
Expand All @@ -157,38 +157,43 @@ public BotToken GetNextToken()
{
case '=':
var nextChar = Read();
return new BotToken(BotTokenType.LessThanEqOperator, inputChar.ToString() + nextChar);
return Token(BotTokenType.LessThanEqOperator, inputChar.ToString() + nextChar);
default:
return new BotToken(BotTokenType.LessThanOperator, inputChar.ToString());
return Token(BotTokenType.LessThanOperator, inputChar.ToString());
}
}
case '$':
{
if (_inputStream.EndOfStream)
return new BotToken(BotTokenType.Error, inputChar.ToString());
return Token(BotTokenType.Error, inputChar.ToString());

// ensure variable starts with letter or underscore before getting variable name
inputChar = (char)_inputStream.Peek();
if (!char.IsLetter(inputChar) && inputChar != '_')
return new BotToken(BotTokenType.Error, inputChar.ToString());
return Token(BotTokenType.Error, inputChar.ToString());

var variable = string.Empty;
for (inputChar = Peek(); !_inputStream.EndOfStream && (char.IsLetterOrDigit(inputChar) || inputChar == '_'); inputChar = Peek())
{
variable += Read();
}

return new BotToken(BotTokenType.Variable, variable);
return Token(BotTokenType.Variable, variable);
}
case '+': return new BotToken(BotTokenType.PlusOperator, inputChar.ToString());
case '-': return new BotToken(BotTokenType.MinusOperator, inputChar.ToString());
case '*': return new BotToken(BotTokenType.MultiplyOperator, inputChar.ToString());
case '/': return new BotToken(BotTokenType.DivideOperator, inputChar.ToString());
default: return new BotToken(BotTokenType.Error, inputChar.ToString());
case '+': return Token(BotTokenType.PlusOperator, inputChar.ToString());
case '-': return Token(BotTokenType.MinusOperator, inputChar.ToString());
case '*': return Token(BotTokenType.MultiplyOperator, inputChar.ToString());
case '/': return Token(BotTokenType.DivideOperator, inputChar.ToString());
default: return Token(BotTokenType.Error, inputChar.ToString());
}
}
}

private BotToken Token(BotTokenType tokenType, string tokenValue)
{
return new BotToken(tokenType, tokenValue, LineNumber, Column);
}

public void Dispose()
{
if (_streamNeedsDispose)
Expand All @@ -199,7 +204,6 @@ public void Dispose()

private char Peek()
{
Column++;
return (char)_inputStream.Peek();
}

Expand Down
4 changes: 2 additions & 2 deletions EOBot/Interpreter/IdentifierBotToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ public class IdentifierBotToken : BotToken
{
public int? ArrayIndex { get; }

public IdentifierBotToken(BotTokenType tokenType, string tokenValue, int? arrayIndex = null)
: base(tokenType, tokenValue)
public IdentifierBotToken(BotTokenType tokenType, string tokenValue, int lineNumber, int column, int? arrayIndex = null)
: base(tokenType, tokenValue, lineNumber, column)
{
ArrayIndex = arrayIndex;
}
Expand Down
2 changes: 1 addition & 1 deletion EOBot/Interpreter/VariableBotToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ public class VariableBotToken : BotToken
public IVariable VariableValue { get; }

public VariableBotToken(BotTokenType tokenType, string tokenValue, IVariable variableValue)
: base(tokenType, tokenValue)
: base(tokenType, tokenValue, 0, 0)
{
VariableValue = variableValue;
}
Expand Down

0 comments on commit a14260c

Please sign in to comment.