Skip to content

Commit

Permalink
Rename check_parser and Identifier
Browse files Browse the repository at this point in the history
  • Loading branch information
jedel1043 committed Jan 29, 2023
1 parent bd0652f commit 9071a5e
Show file tree
Hide file tree
Showing 46 changed files with 331 additions and 299 deletions.
9 changes: 6 additions & 3 deletions boa_parser/src/lexer/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
//! [spec]: https://tc39.es/ecma262/#sec-native-error-types-used-in-this-standard
use boa_ast::Position;
use std::{error::Error as StdError, fmt, io};
use std::{error, fmt, io};

/// An error that occurred during the lexing.
#[derive(Debug)]
Expand All @@ -25,6 +25,7 @@ pub enum Error {
}

impl From<io::Error> for Error {
#[inline]
fn from(err: io::Error) -> Self {
Self::IO(err)
}
Expand All @@ -42,6 +43,7 @@ impl Error {
}

impl fmt::Display for Error {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::IO(e) => write!(f, "I/O error: {e}"),
Expand All @@ -50,8 +52,9 @@ impl fmt::Display for Error {
}
}

impl StdError for Error {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
impl error::Error for Error {
#[inline]
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match self {
Self::IO(err) => Some(err),
Self::Syntax(_, _) => None,
Expand Down
2 changes: 1 addition & 1 deletion boa_parser/src/lexer/identifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ impl<R> Tokenizer<R> for Identifier {
Ok(Keyword::False) => TokenKind::BooleanLiteral(false),
Ok(Keyword::Null) => TokenKind::NullLiteral,
Ok(keyword) => TokenKind::Keyword((keyword, contains_escaped_chars)),
_ => TokenKind::Identifier((
_ => TokenKind::IdentifierName((
interner.get_or_intern(identifier_name.as_str()),
ContainsEscapeSequence(contains_escaped_chars),
)),
Expand Down
6 changes: 3 additions & 3 deletions boa_parser/src/lexer/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,15 +95,15 @@ fn check_identifier() {
TokenKind::identifier(
interner.get_or_intern_static("x\u{200C}\u{200D}", utf16!("x\u{200C}\u{200D}")),
),
TokenKind::Identifier((
TokenKind::IdentifierName((
interner.get_or_intern_static("x", utf16!("x")),
ContainsEscapeSequence(true),
)),
TokenKind::Identifier((
TokenKind::IdentifierName((
interner.get_or_intern_static("xx", utf16!("xx")),
ContainsEscapeSequence(true),
)),
TokenKind::Identifier((
TokenKind::IdentifierName((
interner.get_or_intern_static("xxx", utf16!("xxx")),
ContainsEscapeSequence(true),
)),
Expand Down
42 changes: 34 additions & 8 deletions boa_parser/src/lexer/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,24 @@ pub enum TokenKind {
/// The end of the file.
EOF,

/// An identifier.
Identifier((Sym, ContainsEscapeSequence)),
/// An [**identifier name**][spec].
///
/// [spec]: https://tc39.es/ecma262/#prod-IdentifierName
IdentifierName((Sym, ContainsEscapeSequence)),

/// A private identifier.
/// A [**private identifier**][spec].
///
/// [spec]: https://tc39.es/ecma262/#prod-PrivateIdentifier
PrivateIdentifier(Sym),

/// A keyword and a flag if the keyword contains unicode escaped chars.
///
/// For more information, see [`Keyword`].
Keyword((Keyword, bool)),

/// A `null` literal.
/// The [`null` literal][spec].
///
/// [spec]: https://tc39.es/ecma262/#prod-NullLiteral
NullLiteral,

/// A numeric literal.
Expand All @@ -116,7 +124,9 @@ pub enum TokenKind {
/// A piece of punctuation
Punctuator(Punctuator),

/// A string literal.
/// A [**string literal**][spec].
///
/// [spec]: https://tc39.es/ecma262/#prod-StringLiteral
StringLiteral((Sym, Option<EscapeSequence>)),

/// A part of a template literal without substitution.
Expand All @@ -128,54 +138,63 @@ pub enum TokenKind {
/// A regular expression, consisting of body and flags.
RegularExpressionLiteral(Sym, Sym),

/// Indicates the end of a line (`\n`).
/// Indicates a [**line terminator (`\n`)**][spec].
///
/// [spec]: https://tc39.es/ecma262/#prod-LineTerminator
LineTerminator,

/// Indicates a comment, the content isn't stored.
Comment,
}

impl From<bool> for TokenKind {
#[inline]
fn from(oth: bool) -> Self {
Self::BooleanLiteral(oth)
}
}

impl From<(Keyword, bool)> for TokenKind {
#[inline]
fn from(kw: (Keyword, bool)) -> Self {
Self::Keyword(kw)
}
}

impl From<Punctuator> for TokenKind {
#[inline]
fn from(punc: Punctuator) -> Self {
Self::Punctuator(punc)
}
}

impl From<Numeric> for TokenKind {
#[inline]
fn from(num: Numeric) -> Self {
Self::NumericLiteral(num)
}
}

impl TokenKind {
/// Creates a `BooleanLiteral` token kind.
#[inline]
#[must_use]
pub const fn boolean_literal(lit: bool) -> Self {
Self::BooleanLiteral(lit)
}

/// Creates an `EOF` token kind.
#[inline]
#[must_use]
pub const fn eof() -> Self {
Self::EOF
}

/// Creates an `Identifier` token type.
#[inline]
#[must_use]
pub const fn identifier(ident: Sym) -> Self {
Self::Identifier((ident, ContainsEscapeSequence(false)))
Self::IdentifierName((ident, ContainsEscapeSequence(false)))
}

/// Creates a `NumericLiteral` token kind.
Expand All @@ -187,42 +206,49 @@ impl TokenKind {
}

/// Creates a `Punctuator` token type.
#[inline]
#[must_use]
pub const fn punctuator(punc: Punctuator) -> Self {
Self::Punctuator(punc)
}

/// Creates a `StringLiteral` token type.
#[inline]
#[must_use]
pub const fn string_literal(lit: Sym, escape_sequence: Option<EscapeSequence>) -> Self {
Self::StringLiteral((lit, escape_sequence))
}

/// Creates a `TemplateMiddle` token type.
#[inline]
#[must_use]
pub const fn template_middle(template_string: TemplateString) -> Self {
Self::TemplateMiddle(template_string)
}

/// Creates a `TemplateNoSubstitution` token type.
#[inline]
#[must_use]
pub const fn template_no_substitution(template_string: TemplateString) -> Self {
Self::TemplateNoSubstitution(template_string)
}

/// Creates a `RegularExpressionLiteral` token kind.
#[inline]
#[must_use]
pub const fn regular_expression_literal(body: Sym, flags: Sym) -> Self {
Self::RegularExpressionLiteral(body, flags)
}

/// Creates a `LineTerminator` token kind.
#[inline]
#[must_use]
pub const fn line_terminator() -> Self {
Self::LineTerminator
}

/// Creates a 'Comment' token kind.
#[inline]
#[must_use]
pub const fn comment() -> Self {
Self::Comment
Expand All @@ -234,7 +260,7 @@ impl TokenKind {
match *self {
Self::BooleanLiteral(val) => val.to_string(),
Self::EOF => "end of file".to_owned(),
Self::Identifier((ident, _)) => interner.resolve_expect(ident).to_string(),
Self::IdentifierName((ident, _)) => interner.resolve_expect(ident).to_string(),
Self::PrivateIdentifier(ident) => format!("#{}", interner.resolve_expect(ident)),
Self::Keyword((word, _)) => word.to_string(),
Self::NullLiteral => "null".to_owned(),
Expand Down
5 changes: 3 additions & 2 deletions boa_parser/src/parser/expression/assignment/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ where
.parse(cursor, interner)
}
// ArrowFunction[?In, ?Yield, ?Await] -> ArrowParameters[?Yield, ?Await] -> BindingIdentifier[?Yield, ?Await]
TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => {
TokenKind::IdentifierName(_)
| TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => {
cursor.set_goal(InputElement::Div);

// Because we already peeked the identifier token, there may be a line terminator before the identifier token.
Expand Down Expand Up @@ -146,7 +147,7 @@ where
.or_abrupt()?
&& matches!(
cursor.peek(1, interner).or_abrupt()?.kind(),
TokenKind::Identifier(_)
TokenKind::IdentifierName(_)
| TokenKind::Keyword((Keyword::Yield | Keyword::Await, _))
| TokenKind::Punctuator(Punctuator::OpenParen)
)
Expand Down
2 changes: 1 addition & 1 deletion boa_parser/src/parser/expression/assignment/yield.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ where
.parse(cursor, interner)?;
Ok(Yield::new(Some(expr), true).into())
}
TokenKind::Identifier(_)
TokenKind::IdentifierName(_)
| TokenKind::Punctuator(
Punctuator::OpenParen
| Punctuator::Add
Expand Down
20 changes: 9 additions & 11 deletions boa_parser/src/parser/expression/identifiers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,15 @@ where
let token = cursor.next(interner).or_abrupt()?;

match token.kind() {
TokenKind::Identifier((ident, _))
TokenKind::IdentifierName((ident, _))
if cursor.strict_mode() && RESERVED_IDENTIFIERS_STRICT.contains(ident) =>
{
Err(Error::general(
"using future reserved keyword not allowed in strict mode IdentifierReference",
token.span().start(),
))
}
TokenKind::Identifier((ident, _)) => Ok(Identifier::new(*ident)),
TokenKind::IdentifierName((ident, _)) => Ok(Identifier::new(*ident)),
TokenKind::Keyword((Keyword::Let, _)) if cursor.strict_mode() => Err(Error::general(
"using future reserved keyword not allowed in strict mode IdentifierReference",
token.span().start(),
Expand Down Expand Up @@ -155,19 +155,17 @@ where
let next_token = cursor.next(interner).or_abrupt()?;

match next_token.kind() {
TokenKind::Identifier((Sym::ARGUMENTS, _)) if cursor.strict_mode() => {
TokenKind::IdentifierName((Sym::ARGUMENTS | Sym::EVAL, _)) if cursor.strict_mode() => {
Err(Error::lex(LexError::Syntax(
"unexpected identifier 'arguments' in strict mode".into(),
format!(
"unexpected identifier '{}' in strict mode",
next_token.to_string(interner)
)
.into(),
next_token.span().start(),
)))
}
TokenKind::Identifier((Sym::EVAL, _)) if cursor.strict_mode() => {
Err(Error::lex(LexError::Syntax(
"unexpected identifier 'eval' in strict mode".into(),
next_token.span().start(),
)))
}
TokenKind::Identifier((ident, _)) => {
TokenKind::IdentifierName((ident, _)) => {
if cursor.strict_mode() && RESERVED_IDENTIFIERS_STRICT.contains(ident) {
return Err(Error::general(
"using future reserved keyword not allowed in strict mode",
Expand Down
2 changes: 1 addition & 1 deletion boa_parser/src/parser/expression/left_hand_side/call.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ where
cursor.advance(interner);

let access = match cursor.next(interner).or_abrupt()?.kind() {
TokenKind::Identifier((name, _)) => {
TokenKind::IdentifierName((name, _)) => {
SimplePropertyAccess::new(lhs, *name).into()
}
TokenKind::Keyword((kw, _)) => {
Expand Down
8 changes: 4 additions & 4 deletions boa_parser/src/parser/expression/left_hand_side/member.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,13 @@ where
if cursor.next_if(Punctuator::Dot, interner)?.is_some() {
let token = cursor.next(interner).or_abrupt()?;
match token.kind() {
TokenKind::Identifier((Sym::TARGET, ContainsEscapeSequence(true))) => {
TokenKind::IdentifierName((Sym::TARGET, ContainsEscapeSequence(true))) => {
return Err(Error::general(
"'new.target' must not contain escaped characters",
token.span().start(),
));
}
TokenKind::Identifier((Sym::TARGET, ContainsEscapeSequence(false))) => {
TokenKind::IdentifierName((Sym::TARGET, ContainsEscapeSequence(false))) => {
return Ok(ast::Expression::NewTarget)
}
_ => {
Expand Down Expand Up @@ -122,7 +122,7 @@ where
TokenKind::Punctuator(Punctuator::Dot) => {
let token = cursor.next(interner).or_abrupt()?;
let field = match token.kind() {
TokenKind::Identifier((name, _)) => {
TokenKind::IdentifierName((name, _)) => {
SuperPropertyAccess::new(PropertyAccessField::from(*name))
}
TokenKind::Keyword((kw, _)) => {
Expand Down Expand Up @@ -184,7 +184,7 @@ where
let token = cursor.next(interner).or_abrupt()?;

let access = match token.kind() {
TokenKind::Identifier((name, _)) => {
TokenKind::IdentifierName((name, _)) => {
SimplePropertyAccess::new(lhs, *name).into()
}
TokenKind::Keyword((kw, _)) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,11 @@ where
interner: &mut Interner,
) -> ParseResult<OptionalOperationKind> {
let item = match token.kind() {
TokenKind::Identifier((name, _)) => OptionalOperationKind::SimplePropertyAccess {
field: PropertyAccessField::Const(*name),
},
TokenKind::IdentifierName((name, _)) => {
OptionalOperationKind::SimplePropertyAccess {
field: PropertyAccessField::Const(*name),
}
}
TokenKind::Keyword((kw, _)) => OptionalOperationKind::SimplePropertyAccess {
field: PropertyAccessField::Const(kw.to_sym(interner)),
},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use boa_interner::Interner;
use boa_macros::utf16;

use crate::parser::tests::{check_invalid, check_parser};
use crate::parser::tests::{check_invalid, check_script_parser};
use boa_ast::{
expression::{
access::PropertyAccessField, literal::Literal, Identifier, Optional, OptionalOperation,
Expand All @@ -14,7 +14,7 @@ use boa_ast::{
fn simple() {
let interner = &mut Interner::default();

check_parser(
check_script_parser(
r#"5?.name"#,
vec![Statement::Expression(
Optional::new(
Expand All @@ -40,7 +40,7 @@ fn simple() {
fn complex_chain() {
let interner = &mut Interner::default();

check_parser(
check_script_parser(
r#"a?.b(true)?.["c"]"#,
vec![Statement::Expression(
Optional::new(
Expand Down
Loading

0 comments on commit 9071a5e

Please sign in to comment.