Improve the ergonomics of parsing and peeking

.gitattributes

@@ -0,0 +1,2 @@
+**/generated.rs linguist-generated=true
+**/generated.rs -diff -merge

assets/tokens.yaml

@@ -381,4 +381,4 @@
 - kind: keyword
   variant: Yield
   doc: "The `yield` keyword."
-  keyword: "yield"
+  keyword: "yield"

crates/rune-modules/src/experiments/mod.rs

@@ -21,6 +21,7 @@
 //! ```
 
 use rune::ast;
+use rune::T;
 use rune::{Parser, TokenStream};
 
 mod stringy_math_macro;
@@ -35,7 +36,7 @@ fn make_function(stream: &TokenStream) -> runestick::Result<TokenStream> {
     let mut parser = Parser::from_token_stream(stream);
 
     let ident = parser.parse::<ast::Ident>()?;
-    let _ = parser.parse::<ast::Rocket>()?;
+    let _ = parser.parse::<T![=>]>()?;
     let output = parser.parse::<ast::ExprBlock>()?;
     parser.eof()?;
 

crates/rune-modules/src/test.rs

@@ -20,6 +20,7 @@
 //! # }
 //! ```
 
+use rune::T;
 use rune::ast;
 use rune::macros::stringify;
 use rune::{quote, Parser, TokenStream};
@@ -39,7 +40,7 @@ pub(crate) fn assert_macro(
     let mut parser = Parser::from_token_stream(stream);
     let expr = parser.parse::<ast::Expr>()?;
 
-    let comma = parser.parse::<Option<ast::Comma>>()?;
+    let comma = parser.parse::<Option<T![,]>>()?;
 
     let message = if comma.is_some() {
         parser.parse::<Option<ast::Expr>>()?
@@ -70,10 +71,10 @@ pub(crate) fn assert_eq_macro(
 ) -> runestick::Result<TokenStream> {
     let mut parser = Parser::from_token_stream(stream);
     let left = parser.parse::<ast::Expr>()?;
-    parser.parse::<ast::Comma>()?;
+    parser.parse::<T![,]>()?;
     let right = parser.parse::<ast::Expr>()?;
 
-    let comma = parser.parse::<Option<ast::Comma>>()?;
+    let comma = parser.parse::<Option<T![,]>>()?;
 
     let message = if comma.is_some() {
         parser.parse::<Option<ast::Expr>>()?

crates/rune/src/ast/attribute.rs

@@ -1,22 +1,24 @@
 use crate::ast;
-use crate::{Parse, ParseError, ParseErrorKind, Parser, Peek, Spanned, ToTokens, TokenStream};
+use crate::{
+    Parse, ParseError, ParseErrorKind, Parser, Peek, Peeker, Spanned, ToTokens, TokenStream,
+};
 
 /// Attribute like `#[derive(Debug)]`
 #[derive(Debug, Clone, PartialEq, Eq, ToTokens, Spanned)]
 pub struct Attribute {
     /// The `#` character
-    pub hash: ast::Hash,
+    pub hash: T![#],
     /// Specify if the attribute is outer `#!` or inner `#`
     pub style: AttrStyle,
     /// The `[` character
-    pub open: ast::OpenBracket,
+    pub open: T!['['],
     /// The path of the attribute
     pub path: ast::Path,
     /// The input to the input of the attribute
     #[rune(optional)]
     pub input: TokenStream,
     /// The `]` character
-    pub close: ast::CloseBracket,
+    pub close: T![']'],
 }
 
 /// Parsing an Attribute
@@ -33,23 +35,23 @@ pub struct Attribute {
 /// testing::roundtrip::<ast::Attribute>("#[x+1]");
 /// ```
 impl Parse for Attribute {
-    fn parse(parser: &mut Parser<'_>) -> Result<Self, ParseError> {
-        let hash = parser.parse()?;
-        let style = parser.parse()?;
-        let open = parser.parse()?;
-        let path = parser.parse()?;
+    fn parse(p: &mut Parser<'_>) -> Result<Self, ParseError> {
+        let hash = p.parse()?;
+        let style = p.parse()?;
+        let open = p.parse()?;
+        let path = p.parse()?;
 
         let close;
 
         let mut level = 1;
         let mut input = TokenStream::new();
 
         loop {
-            let token = parser.token_next()?;
+            let token = p.next()?;
 
             match token.kind {
-                ast::Kind::Open(ast::Delimiter::Bracket) => level += 1,
-                ast::Kind::Close(ast::Delimiter::Bracket) => {
+                K!['['] => level += 1,
+                K![']'] => {
                     level -= 1;
                 }
                 _ => (),
@@ -75,13 +77,10 @@ impl Parse for Attribute {
 }
 
 impl Peek for Attribute {
-    fn peek(t1: Option<ast::Token>, t2: Option<ast::Token>) -> bool {
-        let t1 = t1.as_ref().map(|t1| t1.kind);
-        let t2 = t2.as_ref().map(|t2| t2.kind);
-
-        match (t1, t2) {
-            (Some(ast::Kind::Pound), Some(ast::Kind::Bang))
-            | (Some(ast::Kind::Pound), Some(ast::Kind::Open(ast::Delimiter::Bracket))) => true,
+    fn peek(p: &mut Peeker<'_>) -> bool {
+        match (p.nth(0), p.nth(1)) {
+            (K![#], K![!]) => true,
+            (K![#], K!['[']) => true,
             _ => false,
         }
     }
@@ -93,13 +92,13 @@ pub enum AttrStyle {
     /// `#`
     Inner,
     /// `#!`
-    Outer(ast::Bang),
+    Outer(T![!]),
 }
 
 impl Parse for AttrStyle {
-    fn parse(parser: &mut Parser) -> Result<Self, ParseError> {
-        Ok(if parser.peek::<ast::Bang>()? {
-            Self::Outer(parser.parse()?)
+    fn parse(p: &mut Parser) -> Result<Self, ParseError> {
+        Ok(if p.peek::<T![!]>()? {
+            Self::Outer(p.parse()?)
         } else {
             Self::Inner
         })
@@ -110,8 +109,8 @@ impl Parse for AttrStyle {
 pub(crate) struct InnerAttribute(pub(crate) Attribute);
 
 impl Parse for InnerAttribute {
-    fn parse(parser: &mut Parser) -> Result<Self, ParseError> {
-        let attribute: Attribute = parser.parse()?;
+    fn parse(p: &mut Parser) -> Result<Self, ParseError> {
+        let attribute: Attribute = p.parse()?;
 
         match attribute.style {
             AttrStyle::Inner => Ok(Self(attribute)),
@@ -128,12 +127,9 @@ impl Parse for InnerAttribute {
 pub struct OuterAttribute;
 
 impl Peek for OuterAttribute {
-    fn peek(t1: Option<ast::Token>, t2: Option<ast::Token>) -> bool {
-        let kind1 = t1.map(|t| t.kind);
-        let kind2 = t2.map(|t| t.kind);
-
-        match (kind1, kind2) {
-            (Some(ast::Kind::Pound), Some(ast::Kind::Bang)) => true,
+    fn peek(p: &mut Peeker<'_>) -> bool {
+        match (p.nth(0), p.nth(1)) {
+            (K![#], K![!]) => true,
             _ => false,
         }
     }

crates/rune/src/ast/block.rs

@@ -23,11 +23,11 @@ pub struct Block {
     #[rune(id)]
     pub id: Option<Id>,
     /// The close brace.
-    pub open: ast::OpenBrace,
+    pub open: T!['{'],
     /// Statements in the block.
     pub statements: Vec<ast::Stmt>,
     /// The close brace.
-    pub close: ast::CloseBrace,
+    pub close: T!['}'],
 }
 
 impl Opaque for Block {
@@ -88,7 +88,7 @@ impl Parse for Block {
 
         let open = parser.parse()?;
 
-        while !parser.peek::<ast::CloseBrace>()? {
+        while !parser.peek::<T!['}']>()? {
             statements.push(parser.parse()?);
         }
 

crates/rune/src/ast/condition.rs

@@ -21,14 +21,10 @@ pub enum Condition {
 /// testing::roundtrip::<ast::Condition>("let [a, ..] = v");
 /// ```
 impl Parse for Condition {
-    fn parse(parser: &mut Parser) -> Result<Self, ParseError> {
-        let token = parser.token_peek_eof()?;
-
-        Ok(match token.kind {
-            ast::Kind::Let => {
-                Self::ExprLet(Box::new(ast::ExprLet::parse_without_eager_brace(parser)?))
-            }
-            _ => Self::Expr(Box::new(ast::Expr::parse_without_eager_brace(parser)?)),
+    fn parse(p: &mut Parser) -> Result<Self, ParseError> {
+        Ok(match p.nth(0)? {
+            K![let] => Self::ExprLet(Box::new(ast::ExprLet::parse_without_eager_brace(p)?)),
+            _ => Self::Expr(Box::new(ast::Expr::parse_without_eager_brace(p)?)),
         })
     }
 }