diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b7f5cb866..35dfe1f4e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -27,6 +27,7 @@ use core::{ use helpers::attached_token::AttachedToken; use log::debug; +use std::sync::OnceLock; use recursion::RecursionCounter; use IsLateral::*; @@ -3277,6 +3278,12 @@ impl<'a> Parser<'a> { self.peek_nth_token(0) } + /// Return the first non-whitespace token that has not yet been processed + /// (or None if reached end-of-file) + pub fn peek_token_ref(&self) -> &TokenWithLocation { + self.peek_nth_token_ref(0) + } + /// Returns the `N` next non-whitespace tokens that have not yet been /// processed. /// @@ -3304,6 +3311,11 @@ impl<'a> Parser<'a> { .map(|with_loc| with_loc.token) } + pub fn peek_tokens_ref(&self) -> [&Token; N] { + self.peek_tokens_with_location_ref() + .map(|with_loc| &with_loc.token) + } + /// Returns the `N` next non-whitespace tokens with locations that have not /// yet been processed. /// @@ -3327,8 +3339,33 @@ impl<'a> Parser<'a> { }) } + pub fn peek_tokens_with_location_ref(&self) -> [&TokenWithLocation; N] { + let mut index = self.index; + core::array::from_fn(|_| loop { + let token = self.tokens.get(index); + index += 1; + if let Some(TokenWithLocation { + token: Token::Whitespace(_), + span: _, + }) = token + { + continue; + } + if let Some(tok) = token { + return tok; + } else { + return eof_token(); + }; + }) + } + /// Return nth non-whitespace token that has not yet been processed - pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation { + pub fn peek_nth_token(&self, n: usize) -> TokenWithLocation { + self.peek_nth_token_ref(n).clone() + } + + /// Return nth non-whitespace token that has not yet been processed + pub fn peek_nth_token_ref(&self, mut n: usize) -> &TokenWithLocation { let mut index = self.index; loop { index += 1; @@ -3339,10 +3376,11 @@ impl<'a> Parser<'a> { }) => continue, non_whitespace => { if n == 0 { - return non_whitespace.cloned().unwrap_or(TokenWithLocation { - token: Token::EOF, - span: Span::empty(), - }); + if let Some(tok) = non_whitespace { + return tok; + } else { + return eof_token(); + } } n -= 1; } @@ -3379,6 +3417,13 @@ impl<'a> Parser<'a> { /// (or None if reached end-of-file) and mark it as processed. OK to call /// repeatedly after reaching EOF. pub fn next_token(&mut self) -> TokenWithLocation { + self.next_token_ref().clone() + } + + /// Return the first non-whitespace token that has not yet been processed + /// (or None if reached end-of-file) and mark it as processed. OK to call + /// repeatedly after reaching EOF. + pub fn next_token_ref(&mut self) -> &TokenWithLocation { loop { self.index += 1; match self.tokens.get(self.index - 1) { @@ -3387,9 +3432,11 @@ impl<'a> Parser<'a> { span: _, }) => continue, token => { - return token - .cloned() - .unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF)) + if let Some(token) = token { + return token; + } else { + return eof_token(); + } } } } @@ -3442,6 +3489,13 @@ impl<'a> Parser<'a> { } } + pub fn parse_keyword_token_ref(&mut self, expected: Keyword) -> Option<&TokenWithLocation> { + match &self.peek_token_ref().token { + Token::Word(w) if expected == w.keyword => Some(self.next_token_ref()), + _ => None, + } + } + #[must_use] pub fn peek_keyword(&mut self, expected: Keyword) -> bool { matches!(self.peek_token().token, Token::Word(w) if expected == w.keyword) @@ -12855,6 +12909,11 @@ impl Word { } } +static EOF_TOKEN: OnceLock = OnceLock::new(); +fn eof_token() -> &'static TokenWithLocation { + EOF_TOKEN.get_or_init(|| TokenWithLocation::wrap(Token::EOF)) +} + #[cfg(test)] mod tests { use crate::test_utils::{all_dialects, TestedDialects};