From 0f453a5568305496b963c3825e97a356291fd449 Mon Sep 17 00:00:00 2001 From: Coleman McFarland Date: Fri, 6 Apr 2018 18:13:42 -0700 Subject: [PATCH] Refactor to keep up with changes to proc_macro Instead of a `kind` field containting a `TokenNode` variant, a TokenTree is now an enum with variants of different types (Literal, Op, Term, etc). Note that a TokenTree could be a sequence of TokenTrees if it is a Group variant. Other notes: I'm unsure about the set_span call in Builder::emit_if, but I did not want to throw away the passed in Span. Parsing relies on destructuring references to the values associated with TokenTree enum variants It doesn't seem as easy to compose/chain TokenStreams as it is to collect a Vec into a TokenStream. There is probably some iterator API I could use instead. See `match_arms` and build.rs Refs #121 --- maud_macros/src/build.rs | 33 ++++-- maud_macros/src/lib.rs | 17 +-- maud_macros/src/parse.rs | 216 +++++++++++++++++++-------------------- 3 files changed, 139 insertions(+), 127 deletions(-) diff --git a/maud_macros/src/build.rs b/maud_macros/src/build.rs index 75d5db7a..97666185 100644 --- a/maud_macros/src/build.rs +++ b/maud_macros/src/build.rs @@ -1,6 +1,5 @@ -use proc_macro::{Delimiter, Literal, Span, TokenNode, TokenStream, TokenTree}; +use proc_macro::{Delimiter, Group, Literal, Span, TokenStream, TokenTree}; use proc_macro::quote; - use maud_htmlescape::Escaper; pub struct Builder { @@ -24,7 +23,7 @@ impl Builder { if !self.tail.is_empty() { let expr = { let output_ident = self.output_ident.clone(); - let string = TokenNode::Literal(Literal::string(&self.tail)); + let string = TokenTree::Literal(Literal::string(&self.tail)); quote!($output_ident.push_str($string);) }; self.stmts.push(expr); @@ -35,7 +34,15 @@ impl Builder { /// Reifies the `Builder` into a raw list of statements. pub fn build(mut self) -> TokenStream { let Builder { stmts, .. } = { self.flush(); self }; - stmts.into_iter().collect() + + // use a Group here? + let mut tts: Vec = Vec::new(); + for s in stmts.into_iter() { + let i = s.into_iter(); + tts.extend(i); + } + + tts.into_iter().collect() } /// Pushes a statement, flushing the tail buffer in the process. @@ -111,14 +118,20 @@ impl Builder { ) { // If the condition contains an opening brace `{`, // wrap it in parentheses to avoid parse errors - if cond.clone().into_iter().any(|token| match token.kind { - TokenNode::Group(Delimiter::Brace, _) => true, + if cond.clone().into_iter().any(|token| match token { + TokenTree::Group(grp) => { + if grp.delimiter() == Delimiter::Brace { + true + } else { + false + } + }, _ => false, }) { - cond = TokenStream::from(TokenTree { - kind: TokenNode::Group(Delimiter::Parenthesis, cond), - span: cond_span, - }); + let mut g = Group::new(Delimiter::Parenthesis, cond); + // NOTE: Do we need to do this? + g.set_span(cond_span); + cond = TokenStream::from(TokenTree::Group(g)); } self.push(quote!(if $cond { $body })); } diff --git a/maud_macros/src/lib.rs b/maud_macros/src/lib.rs index 1045fd3f..0a5f2be1 100644 --- a/maud_macros/src/lib.rs +++ b/maud_macros/src/lib.rs @@ -1,4 +1,5 @@ #![feature(proc_macro)] +#![feature(pattern_parentheses)] #![doc(html_root_url = "https://docs.rs/maud_macros/0.17.2")] @@ -9,7 +10,7 @@ extern crate proc_macro; mod parse; mod build; -use proc_macro::{Literal, Span, Term, TokenNode, TokenStream, TokenTree}; +use proc_macro::{Span, Term, TokenStream, TokenTree}; use proc_macro::quote; type ParseResult = Result; @@ -27,21 +28,21 @@ pub fn html_debug(input: TokenStream) -> TokenStream { } fn expand(input: TokenStream) -> TokenStream { - let output_ident = TokenTree { - kind: TokenNode::Term(Term::intern("__maud_output")), - span: Span::def_site(), - }; + + let output_ident = TokenTree::Term(Term::new("__maud_output", Span::def_site())); // Heuristic: the size of the resulting markup tends to correlate with the // code size of the template itself - let size_hint = input.to_string().len(); - let size_hint = TokenNode::Literal(Literal::u64(size_hint as u64)); + // + // NOTE: can't get this to compile inside quote! + //let size_hint = Literal::u64_unsuffixed(size_hint as u64); let stmts = match parse::parse(input, output_ident.clone()) { Ok(stmts) => stmts, Err(e) => panic!(e), }; quote!({ extern crate maud; - let mut $output_ident = String::with_capacity($size_hint as usize); + //let mut $output_ident = String::with_capacity($size_hint as usize); + let mut $output_ident = String::new(); $stmts maud::PreEscaped($output_ident) }) diff --git a/maud_macros/src/parse.rs b/maud_macros/src/parse.rs index c9bdbdf1..aaece529 100644 --- a/maud_macros/src/parse.rs +++ b/maud_macros/src/parse.rs @@ -1,14 +1,15 @@ use proc_macro::{ Delimiter, + Group, Literal, Spacing, Span, - TokenNode, + Term, TokenStream, TokenTree, - TokenTreeIter, }; -use std::iter; + +use proc_macro::token_stream; use std::mem; use literalext::LiteralExt; @@ -28,7 +29,7 @@ struct Parser { output_ident: TokenTree, /// Indicates whether we're inside an attribute node. in_attr: bool, - input: TokenTreeIter, + input: token_stream::IntoIter, } impl Iterator for Parser { @@ -97,24 +98,18 @@ impl Parser { loop { match self.peek2() { None => return Ok(()), - Some((TokenTree { kind: TokenNode::Op(';', _), .. }, _)) => self.advance(), - Some(( - TokenTree { kind: TokenNode::Op('@', _), .. }, - Some(TokenTree { kind: TokenNode::Term(term), span }), - )) if term.as_str() == "let" => { + Some((TokenTree::Op(o), _)) if o.op() == ';' => self.advance(), + Some((TokenTree::Op(o), Some(TokenTree::Term(term)))) if o.op() == '@' && term.as_str() == "let" => { // When emitting a `@let`, wrap the rest of the block in a // new block to avoid scoping issues - let keyword = TokenTree { kind: TokenNode::Term(term), span }; + let keyword = Term::new(term.as_str(), term.span()); self.advance2(); builder.push({ let mut builder = self.builder(); - builder.push(keyword); + builder.push(TokenTree::Term(keyword)); self.let_expr(&mut builder)?; self.markups(&mut builder)?; - TokenTree { - kind: TokenNode::Group(Delimiter::Brace, builder.build()), - span, - } + TokenTree::Group(Group::new(Delimiter::Brace, builder.build())) }); }, _ => self.markup(builder)?, @@ -130,17 +125,17 @@ impl Parser { }; match token { // Literal - TokenTree { kind: TokenNode::Literal(lit), .. } => { + TokenTree::Literal(lit) => { self.advance(); self.literal(lit, builder)?; }, // Special form - TokenTree { kind: TokenNode::Op('@', _), .. } => { + TokenTree::Op(o) if o.op() == '@' => { self.advance(); match self.next() { - Some(TokenTree { kind: TokenNode::Term(term), span }) => { - let keyword = TokenTree { kind: TokenNode::Term(term), span }; - builder.push(keyword); + Some(TokenTree::Term(term)) => { + let keyword = Term::new(term.as_str(), term.span()); + builder.push(TokenTree::Term(keyword)); match term.as_str() { "if" => self.if_expr(builder)?, "while" => self.while_expr(builder)?, @@ -152,21 +147,21 @@ impl Parser { }, _ => return self.error("expected keyword after `@`"), } - } + }, // Element - TokenTree { kind: TokenNode::Term(_), .. } => { + TokenTree::Term(_) => { let name = self.namespaced_name()?; self.element(&name, builder)?; }, // Splice - TokenTree { kind: TokenNode::Group(Delimiter::Parenthesis, expr), .. } => { + TokenTree::Group(ref grp) if grp.delimiter() == Delimiter::Parenthesis => { self.advance(); - builder.splice(expr); - } + builder.splice(grp.stream()); + }, // Block - TokenTree { kind: TokenNode::Group(Delimiter::Brace, block), .. } => { + TokenTree::Group(ref grp) if grp.delimiter() == Delimiter::Brace => { self.advance(); - self.with_input(block).markups(builder)?; + self.with_input(grp.stream()).markups(builder)?; }, // ??? _ => return self.error("invalid syntax"), @@ -190,8 +185,9 @@ impl Parser { fn if_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { loop { match self.next() { - Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, block), span }) => { - let block = self.block(block, span)?; + Some(TokenTree::Group(ref block)) + if block.delimiter() == Delimiter::Brace => { + let block = self.block(block.stream(), block.span())?; builder.push(block); break; }, @@ -207,30 +203,28 @@ impl Parser { /// The leading `@else if` or `@else` should *not* already be consumed. fn else_if_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { match self.peek2() { - // Try to match an `@else` after this - Some(( - TokenTree { kind: TokenNode::Op('@', _), .. }, - Some(TokenTree { kind: TokenNode::Term(else_keyword), span }), - )) if else_keyword.as_str() == "else" => { + Some((TokenTree::Op(o), Some(TokenTree::Term(else_keyword)))) + if o.op() == '@' && else_keyword.as_str() == "else" => { self.advance2(); - let else_keyword = TokenTree { kind: TokenNode::Term(else_keyword), span }; - builder.push(else_keyword); + let else_keyword = Term::new("else", else_keyword.span()); + builder.push(TokenTree::Term(else_keyword)); match self.peek() { // `@else if` - Some(TokenTree { kind: TokenNode::Term(if_keyword), span }) - if if_keyword.as_str() == "if" => { + Some(TokenTree::Term(if_keyword)) if if_keyword.as_str() == "if" => { self.advance(); - let if_keyword = TokenTree { kind: TokenNode::Term(if_keyword), span }; - builder.push(if_keyword); + let if_keyword = Term::new("if", if_keyword.span()); + builder.push(TokenTree::Term(if_keyword)); self.if_expr(builder)?; }, // Just an `@else` _ => { - if let Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, block), span }) = self.next() { - let block = self.block(block, span)?; - builder.push(block); - } else { - return self.error("expected body for @else"); + // match brace `{` + match self.next() { + Some(TokenTree::Group(ref grp)) if grp.delimiter() == Delimiter::Brace => { + let block = self.block(grp.stream(), grp.span())?; + builder.push(block); + }, + _ => { return self.error("expected body for @else"); }, } }, } @@ -247,8 +241,8 @@ impl Parser { fn while_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { loop { match self.next() { - Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, block), span }) => { - let block = self.block(block, span)?; + Some(TokenTree::Group(ref block)) if block.delimiter() == Delimiter::Brace => { + let block = self.block(block.stream(), block.span())?; builder.push(block); break; }, @@ -265,8 +259,9 @@ impl Parser { fn for_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { loop { match self.next() { - Some(TokenTree { kind: TokenNode::Term(in_keyword), span }) if in_keyword.as_str() == "in" => { - builder.push(TokenTree { kind: TokenNode::Term(in_keyword), span }); + Some(TokenTree::Term(in_keyword)) if in_keyword.as_str() == "in" => { + let in_keyword = Term::new("in", in_keyword.span()); + builder.push(TokenTree::Term(in_keyword)); break; }, Some(token) => builder.push(token), @@ -275,8 +270,8 @@ impl Parser { } loop { match self.next() { - Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, block), span }) => { - let block = self.block(block, span)?; + Some(TokenTree::Group(ref block)) if block.delimiter() == Delimiter::Brace => { + let block = self.block(block.stream(), block.span())?; builder.push(block); break; }, @@ -293,12 +288,10 @@ impl Parser { fn match_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { loop { match self.next() { - Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, body), span }) => { - let body = self.with_input(body).match_arms()?; - builder.push(TokenTree { - kind: TokenNode::Group(Delimiter::Brace, body), - span, - }); + Some(TokenTree::Group(ref body)) if body.delimiter() == Delimiter::Brace => { + let body = self.with_input(body.stream()).match_arms()?; + let body = Group::new(Delimiter::Brace, body); + builder.push(TokenTree::Group(body)); break; }, Some(token) => builder.push(token), @@ -309,24 +302,22 @@ impl Parser { } fn match_arms(&mut self) -> ParseResult { - let mut arms = Vec::new(); + let mut arms: Vec = Vec::new(); while let Some(arm) = self.match_arm()? { - arms.push(arm); + arms.extend(arm); } Ok(arms.into_iter().collect()) } - fn match_arm(&mut self) -> ParseResult> { - let mut pat = Vec::new(); + fn match_arm(&mut self) -> ParseResult>> { + let mut pat: Vec = Vec::new(); loop { match self.peek2() { - Some(( - eq @ TokenTree { kind: TokenNode::Op('=', Spacing::Joint), .. }, - Some(gt @ TokenTree { kind: TokenNode::Op('>', _), .. }), - )) => { + Some((TokenTree::Op(eq), Some(TokenTree::Op(gt)))) + if eq.op() == '=' && gt.op() == '>' && eq.spacing() == Spacing::Joint => { self.advance2(); - pat.push(eq); - pat.push(gt); + pat.push(TokenTree::Op(eq)); + pat.push(TokenTree::Op(gt)); break; }, Some((token, _)) => { @@ -343,23 +334,25 @@ impl Parser { } let body = match self.next() { // $pat => { $stmts } - Some(TokenTree { kind: TokenNode::Group(Delimiter::Brace, body), span }) => { - let body = self.block(body, span)?; + Some(TokenTree::Group(ref body)) if body.delimiter() == Delimiter::Brace => { + let body: TokenTree = self.block(body.stream(), body.span())?; // Trailing commas are optional if the match arm is a braced block - if let Some(TokenTree { kind: TokenNode::Op(',', _), .. }) = self.peek() { - self.advance(); + if let Some(TokenTree::Op(o)) = self.peek() { + if o.op() == ',' { + self.advance(); + } } body }, // $pat => $expr Some(first_token) => { - let mut span = first_token.span; + let mut span = first_token.span(); let mut body = vec![first_token]; loop { match self.next() { - Some(TokenTree { kind: TokenNode::Op(',', _), .. }) => break, + Some(TokenTree::Op(o)) if o.op() == ',' => break, Some(token) => { - if let Some(bigger_span) = span.join(token.span) { + if let Some(bigger_span) = span.join(token.span()) { span = bigger_span; } body.push(token); @@ -371,7 +364,8 @@ impl Parser { }, None => return self.error("unexpected end of @match arm"), }; - Ok(Some(pat.into_iter().chain(iter::once(body)).collect())) + pat.push(body); + Ok(Some(pat)) } /// Parses and renders a `@let` expression. @@ -380,21 +374,29 @@ impl Parser { fn let_expr(&mut self, builder: &mut Builder) -> ParseResult<()> { loop { match self.next() { - Some(token @ TokenTree { kind: TokenNode::Op('=', _), .. }) => { - builder.push(token); - break; + Some(token) => { + match token { + TokenTree::Op(ref o) if o.op() == '=' => { + builder.push(token.clone()); + break; + } + _ => builder.push(token), + } }, - Some(token) => builder.push(token), None => return self.error("unexpected end of @let expression"), } } loop { match self.next() { - Some(token @ TokenTree { kind: TokenNode::Op(';', _), .. }) => { - builder.push(token); - break; + Some(token) => { + match token { + TokenTree::Op(ref o) if o.op() == ';' => { + builder.push(token.clone()); + break; + }, + _ => builder.push(token), + } }, - Some(token) => builder.push(token), None => return self.error("unexpected end of @let expression"), } } @@ -412,8 +414,7 @@ impl Parser { self.attrs(builder)?; builder.element_open_end(); match self.peek() { - Some(TokenTree { kind: TokenNode::Op(';', _), .. }) | - Some(TokenTree { kind: TokenNode::Op('/', _), .. }) => { + Some(TokenTree::Op(o)) if o.op() == ';' || o.op() == '/' => { // Void element self.advance(); }, @@ -436,7 +437,7 @@ impl Parser { let token_after = attempt.next(); match (maybe_name, token_after) { // Non-empty attribute - (Ok(name), Some(TokenTree { kind: TokenNode::Op('=', _), .. })) => { + (Ok(ref name), Some(TokenTree::Op(ref o))) if o.op() == '=' => { self.commit(attempt); builder.attribute_start(&name); { @@ -448,7 +449,7 @@ impl Parser { builder.attribute_end(); }, // Empty attribute - (Ok(name), Some(TokenTree { kind: TokenNode::Op('?', _), .. })) => { + (Ok(ref name), Some(TokenTree::Op(ref o))) if o.op() == '?' => { self.commit(attempt); if let Some((cond, cond_span)) = self.attr_toggler() { // Toggle the attribute based on a boolean expression @@ -464,7 +465,7 @@ impl Parser { } }, // Class shorthand - (Err(_), Some(TokenTree { kind: TokenNode::Op('.', _), .. })) => { + (Err(_), Some(TokenTree::Op(o))) if o.op() == '.' => { self.commit(attempt); let class_name = self.name()?; if let Some((cond, cond_span)) = self.attr_toggler() { @@ -476,7 +477,7 @@ impl Parser { } }, // ID shorthand - (Err(_), Some(TokenTree { kind: TokenNode::Op('#', _), .. })) => { + (Err(_), Some(TokenTree::Op(o))) if o.op() == '#' => { self.commit(attempt); ids.push(self.name()?); }, @@ -512,20 +513,18 @@ impl Parser { /// Parses the `[cond]` syntax after an empty attribute or class shorthand. fn attr_toggler(&mut self) -> Option<(TokenStream, Span)> { - if let Some(TokenTree { - kind: TokenNode::Group(Delimiter::Bracket, cond), - span: delim_span, - }) = self.peek() { - self.advance(); - Some((cond, delim_span)) - } else { - None + match self.peek() { + Some(TokenTree::Group(ref grp)) if grp.delimiter() == Delimiter::Bracket => { + self.advance(); + Some((grp.stream(), grp.span())) + } + _ => None } } /// Parses an identifier, without dealing with namespaces. fn name(&mut self) -> ParseResult { - let mut s = if let Some(TokenTree { kind: TokenNode::Term(term), .. }) = self.peek() { + let mut s = if let Some(TokenTree::Term(term)) = self.peek() { self.advance(); String::from(term.as_str()) } else { @@ -534,12 +533,12 @@ impl Parser { let mut expect_ident = false; loop { expect_ident = match self.peek() { - Some(TokenTree { kind: TokenNode::Op('-', _), .. }) => { + Some(TokenTree::Op(o)) if o.op() == '-' => { self.advance(); s.push('-'); true }, - Some(TokenTree { kind: TokenNode::Term(term), .. }) if expect_ident => { + Some(TokenTree::Term(term)) if expect_ident => { self.advance(); s.push_str(term.as_str()); false @@ -554,22 +553,21 @@ impl Parser { /// if necessary. fn namespaced_name(&mut self) -> ParseResult { let mut s = self.name()?; - if let Some(TokenTree { kind: TokenNode::Op(':', _), .. }) = self.peek() { - self.advance(); - s.push(':'); - s.push_str(&self.name()?); + if let Some(TokenTree::Op(o)) = self.peek() { + if o.op() == ':' { + self.advance(); + s.push(':'); + s.push_str(&self.name()?); + } } Ok(s) } /// Parses the given token stream as a Maud expression, returning a block of /// Rust code. - fn block(&mut self, body: TokenStream, span: Span) -> ParseResult { + fn block(&mut self, body: TokenStream, _span: Span) -> ParseResult { let mut builder = self.builder(); self.with_input(body).markups(&mut builder)?; - Ok(TokenTree { - kind: TokenNode::Group(Delimiter::Brace, builder.build()), - span, - }) + Ok(TokenTree::Group(Group::new(Delimiter::Brace, builder.build()))) } }