From 783f8e26a2222228f3212e80b43eea3ced8eb5cd Mon Sep 17 00:00:00 2001 From: creampnx_x <2270436024@qq.com> Date: Sat, 24 Sep 2022 15:16:01 +0800 Subject: [PATCH 1/3] fix: add unicode terminator to line comment --- boa_engine/src/syntax/lexer/comment.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/boa_engine/src/syntax/lexer/comment.rs b/boa_engine/src/syntax/lexer/comment.rs index 1e5ecdf367e..55d94dac305 100644 --- a/boa_engine/src/syntax/lexer/comment.rs +++ b/boa_engine/src/syntax/lexer/comment.rs @@ -34,12 +34,14 @@ impl Tokenizer for SingleLineComment { let _timer = Profiler::global().start_event("SingleLineComment", "Lexing"); // Skip either to the end of the line or to the end of the input - while let Some(ch) = cursor.peek()? { - if ch == b'\n' || ch == b'\r' { - break; - } - // Consume char. - cursor.next_byte()?.expect("Comment character vanished"); + while let Some(ch) = cursor.next_char()? { + let tried_ch = char::try_from(ch); + match tried_ch { + Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => { + break; + } + _ => {} + }; } Ok(Token::new( TokenKind::Comment, From 20f3c49b61ac93b308c282411f866502f8c800ae Mon Sep 17 00:00:00 2001 From: creampnx_x <2270436024@qq.com> Date: Sat, 24 Sep 2022 15:24:41 +0800 Subject: [PATCH 2/3] format-code --- boa_engine/src/syntax/lexer/comment.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/boa_engine/src/syntax/lexer/comment.rs b/boa_engine/src/syntax/lexer/comment.rs index 55d94dac305..9bd2a91a4b1 100644 --- a/boa_engine/src/syntax/lexer/comment.rs +++ b/boa_engine/src/syntax/lexer/comment.rs @@ -37,9 +37,7 @@ impl Tokenizer for SingleLineComment { while let Some(ch) = cursor.next_char()? { let tried_ch = char::try_from(ch); match tried_ch { - Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => { - break; - } + Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => break, _ => {} }; } From a70520f41598572718235ce9bcbe966f53764493 Mon Sep 17 00:00:00 2001 From: creampnx_x <2270436024@qq.com> Date: Sat, 24 Sep 2022 16:29:57 +0800 Subject: [PATCH 3/3] fix: use peek_char instead of next_char --- boa_engine/src/syntax/lexer/comment.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boa_engine/src/syntax/lexer/comment.rs b/boa_engine/src/syntax/lexer/comment.rs index 9bd2a91a4b1..a34bce9fbca 100644 --- a/boa_engine/src/syntax/lexer/comment.rs +++ b/boa_engine/src/syntax/lexer/comment.rs @@ -34,12 +34,13 @@ impl Tokenizer for SingleLineComment { let _timer = Profiler::global().start_event("SingleLineComment", "Lexing"); // Skip either to the end of the line or to the end of the input - while let Some(ch) = cursor.next_char()? { + while let Some(ch) = cursor.peek_char()? { let tried_ch = char::try_from(ch); match tried_ch { Ok(c) if c == '\r' || c == '\n' || c == '\u{2028}' || c == '\u{2029}' => break, _ => {} }; + cursor.next_char().expect("Comment character vanished"); } Ok(Token::new( TokenKind::Comment,