diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index ac8173f101a65..e13f6cabb5296 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -1003,7 +1003,7 @@ impl<'a> LoweringContext<'a> { AttrKind::Normal(ref item) => { AttrKind::Normal(AttrItem { path: item.path.clone(), - tokens: self.lower_token_stream(item.tokens.clone()), + args: self.lower_mac_args(&item.args), }) } AttrKind::DocComment(comment) => AttrKind::DocComment(comment) @@ -1017,6 +1017,16 @@ impl<'a> LoweringContext<'a> { } } + fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs { + match *args { + MacArgs::Empty => MacArgs::Empty, + MacArgs::Delimited(dspan, delim, ref tokens) => + MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone())), + MacArgs::Eq(eq_span, ref tokens) => + MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone())), + } + } + fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream { tokens .into_trees() diff --git a/src/librustc/hir/lowering/item.rs b/src/librustc/hir/lowering/item.rs index f689e7f96222f..ff9d8c85df8b9 100644 --- a/src/librustc/hir/lowering/item.rs +++ b/src/librustc/hir/lowering/item.rs @@ -233,7 +233,7 @@ impl LoweringContext<'_> { if let ItemKind::MacroDef(ref def) = i.kind { if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) { - let body = self.lower_token_stream(def.stream()); + let body = self.lower_token_stream(def.body.inner_tokens()); let hir_id = self.lower_node_id(i.id); self.exported_macros.push(hir::MacroDef { name: ident.name, diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs index b08a095beac4e..0fd7145f425d3 100644 --- a/src/librustc_lint/builtin.rs +++ b/src/librustc_lint/builtin.rs @@ -1450,10 +1450,10 @@ impl KeywordIdents { impl EarlyLintPass for KeywordIdents { fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef, _id: ast::NodeId) { - self.check_tokens(cx, mac_def.stream()); + self.check_tokens(cx, mac_def.body.inner_tokens()); } fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) { - self.check_tokens(cx, mac.tts.clone().into()); + self.check_tokens(cx, mac.args.inner_tokens()); } fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: ast::Ident) { self.check_ident_token(cx, UnderMacro(false), ident); diff --git a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs index 8214153f153f1..13db9a6fef9ca 100644 --- a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs +++ b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs @@ -32,6 +32,8 @@ use syntax::source_map; use syntax::source_map::Spanned; use syntax::symbol::Symbol; use syntax::expand::allocator::AllocatorKind; +use syntax::ptr::P; +use syntax::tokenstream::DelimSpan; use syntax_pos::{Span, FileName}; macro_rules! provide { @@ -427,6 +429,7 @@ impl CStore { let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body); let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos); + let dspan = DelimSpan::from_single(local_span); let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None); emit_unclosed_delims(&mut errors, &sess.parse_sess); @@ -448,7 +451,7 @@ impl CStore { span: local_span, attrs: attrs.iter().cloned().collect(), kind: ast::ItemKind::MacroDef(ast::MacroDef { - tokens: body.into(), + body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)), legacy: def.legacy, }), vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited), diff --git a/src/librustc_parse/config.rs b/src/librustc_parse/config.rs index 26e51e83d625a..1bf6e9ecbc060 100644 --- a/src/librustc_parse/config.rs +++ b/src/librustc_parse/config.rs @@ -101,7 +101,7 @@ impl<'a> StripUnconfigured<'a> { if !attr.has_name(sym::cfg_attr) { return vec![attr]; } - if attr.get_normal_item().tokens.is_empty() { + if let ast::MacArgs::Empty = attr.get_normal_item().args { self.sess.span_diagnostic .struct_span_err( attr.span, diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs index 1215c7a199a98..a22b383e5f391 100644 --- a/src/librustc_parse/lib.rs +++ b/src/librustc_parse/lib.rs @@ -277,7 +277,9 @@ pub fn parse_in_attr<'a, T>( ) -> PResult<'a, T> { let mut parser = Parser::new( sess, - attr.get_normal_item().tokens.clone(), + // FIXME(#66940, Centril | petrochenkov): refactor this function so it doesn't + // require reconstructing and immediately re-parsing delimiters. + attr.get_normal_item().args.outer_tokens(), None, false, false, @@ -409,7 +411,7 @@ fn prepend_attrs( brackets.push(stream); } - brackets.push(item.tokens.clone()); + brackets.push(item.args.outer_tokens()); // The span we list here for `#` and for `[ ... ]` are both wrong in // that it encompasses more than each token, but it hopefully is "good diff --git a/src/librustc_parse/parser/attr.rs b/src/librustc_parse/parser/attr.rs index 524b551e54cb3..c7261404f54ef 100644 --- a/src/librustc_parse/parser/attr.rs +++ b/src/librustc_parse/parser/attr.rs @@ -2,8 +2,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle}; use syntax::attr; use syntax::ast; use syntax::util::comments; -use syntax::token::{self, Nonterminal, DelimToken}; -use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::token::{self, Nonterminal}; use syntax_pos::{Span, Symbol}; use errors::PResult; @@ -181,31 +180,8 @@ impl<'a> Parser<'a> { item } else { let path = self.parse_path(PathStyle::Mod)?; - let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) || - self.check(&token::OpenDelim(DelimToken::Bracket)) || - self.check(&token::OpenDelim(DelimToken::Brace)) { - self.parse_token_tree().into() - } else if self.eat(&token::Eq) { - let eq = TokenTree::token(token::Eq, self.prev_span); - let mut is_interpolated_expr = false; - if let token::Interpolated(nt) = &self.token.kind { - if let token::NtExpr(..) = **nt { - is_interpolated_expr = true; - } - } - let token_tree = if is_interpolated_expr { - // We need to accept arbitrary interpolated expressions to continue - // supporting things like `doc = $expr` that work on stable. - // Non-literal interpolated expressions are rejected after expansion. - self.parse_token_tree() - } else { - self.parse_unsuffixed_lit()?.token_tree() - }; - TokenStream::new(vec![eq.into(), token_tree.into()]) - } else { - TokenStream::default() - }; - ast::AttrItem { path, tokens } + let args = self.parse_attr_args()?; + ast::AttrItem { path, args } }) } @@ -244,7 +220,7 @@ impl<'a> Parser<'a> { Ok(attrs) } - fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> { + pub(super) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> { let lit = self.parse_lit()?; debug!("checking if {:?} is unusuffixed", lit); diff --git a/src/librustc_parse/parser/expr.rs b/src/librustc_parse/parser/expr.rs index 43c740f7f93f1..1112274dc46a5 100644 --- a/src/librustc_parse/parser/expr.rs +++ b/src/librustc_parse/parser/expr.rs @@ -922,13 +922,11 @@ impl<'a> Parser<'a> { // `!`, as an operator, is prefix, so we know this isn't that. if self.eat(&token::Not) { // MACRO INVOCATION expression - let (delim, tts) = self.expect_delimited_token_tree()?; + let args = self.parse_mac_args()?; hi = self.prev_span; ex = ExprKind::Mac(Mac { path, - tts, - delim, - span: lo.to(hi), + args, prior_type_ascription: self.last_type_ascription, }); } else if self.check(&token::OpenDelim(token::Brace)) { diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs index a0669a2a1748e..46addba57c628 100644 --- a/src/librustc_parse/parser/item.rs +++ b/src/librustc_parse/parser/item.rs @@ -8,12 +8,12 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit}; use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind}; use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField}; -use syntax::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param}; +use syntax::ast::{Mac, MacArgs, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param}; use syntax::print::pprust; use syntax::ptr::P; use syntax::ThinVec; use syntax::token; -use syntax::tokenstream::{TokenTree, TokenStream}; +use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream}; use syntax::source_map::{self, respan, Span}; use syntax::struct_span_err; use syntax_pos::BytePos; @@ -432,22 +432,18 @@ impl<'a> Parser<'a> { let prev_span = self.prev_span; self.complain_if_pub_macro(&visibility.node, prev_span); - let mac_lo = self.token.span; - // Item macro let path = self.parse_path(PathStyle::Mod)?; self.expect(&token::Not)?; - let (delim, tts) = self.expect_delimited_token_tree()?; - if delim != MacDelimiter::Brace && !self.eat(&token::Semi) { + let args = self.parse_mac_args()?; + if args.need_semicolon() && !self.eat(&token::Semi) { self.report_invalid_macro_expansion_item(); } let hi = self.prev_span; let mac = Mac { path, - tts, - delim, - span: mac_lo.to(hi), + args, prior_type_ascription: self.last_type_ascription, }; let item = @@ -500,7 +496,6 @@ impl<'a> Parser<'a> { if self.token.is_path_start() && !(self.is_async_fn() && self.token.span.rust_2015()) { let prev_span = self.prev_span; - let lo = self.token.span; let path = self.parse_path(PathStyle::Mod)?; if path.segments.len() == 1 { @@ -518,16 +513,14 @@ impl<'a> Parser<'a> { *at_end = true; // eat a matched-delimiter token tree: - let (delim, tts) = self.expect_delimited_token_tree()?; - if delim != MacDelimiter::Brace { + let args = self.parse_mac_args()?; + if args.need_semicolon() { self.expect_semi()?; } Ok(Some(Mac { path, - tts, - delim, - span: lo.to(self.prev_span), + args, prior_type_ascription: self.last_type_ascription, })) } else { @@ -1624,33 +1617,31 @@ impl<'a> Parser<'a> { vis: &Visibility, lo: Span ) -> PResult<'a, Option>> { - let token_lo = self.token.span; let (ident, def) = if self.eat_keyword(kw::Macro) { let ident = self.parse_ident()?; - let tokens = if self.check(&token::OpenDelim(token::Brace)) { - match self.parse_token_tree() { - TokenTree::Delimited(_, _, tts) => tts, - _ => unreachable!(), - } + let body = if self.check(&token::OpenDelim(token::Brace)) { + self.parse_mac_args()? } else if self.check(&token::OpenDelim(token::Paren)) { - let args = self.parse_token_tree(); + let params = self.parse_token_tree(); + let pspan = params.span(); let body = if self.check(&token::OpenDelim(token::Brace)) { self.parse_token_tree() } else { - self.unexpected()?; - unreachable!() + return self.unexpected(); }; - TokenStream::new(vec![ - args.into(), - TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(), + let bspan = body.span(); + let tokens = TokenStream::new(vec![ + params.into(), + TokenTree::token(token::FatArrow, pspan.between(bspan)).into(), body.into(), - ]) + ]); + let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); + P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens)) } else { - self.unexpected()?; - unreachable!() + return self.unexpected(); }; - (ident, ast::MacroDef { tokens: tokens.into(), legacy: false }) + (ident, ast::MacroDef { body, legacy: false }) } else if self.check_keyword(sym::macro_rules) && self.look_ahead(1, |t| *t == token::Not) && self.look_ahead(2, |t| t.is_ident()) { @@ -1660,12 +1651,12 @@ impl<'a> Parser<'a> { self.bump(); let ident = self.parse_ident()?; - let (delim, tokens) = self.expect_delimited_token_tree()?; - if delim != MacDelimiter::Brace && !self.eat(&token::Semi) { + let body = self.parse_mac_args()?; + if body.need_semicolon() && !self.eat(&token::Semi) { self.report_invalid_macro_expansion_item(); } - (ident, ast::MacroDef { tokens, legacy: true }) + (ident, ast::MacroDef { body, legacy: true }) } else { return Ok(None); }; diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index ea7673767d07e..28689720044e8 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -16,7 +16,7 @@ use crate::lexer::UnmatchedBrace; use syntax::ast::{ self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Extern, Ident, StrLit, - IsAsync, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety, + IsAsync, MacArgs, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety, }; use syntax::print::pprust; @@ -1010,27 +1010,49 @@ impl<'a> Parser<'a> { } } - fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> { - let delim = match self.token.kind { - token::OpenDelim(delim) => delim, - _ => { - let msg = "expected open delimiter"; - let mut err = self.fatal(msg); - err.span_label(self.token.span, msg); - return Err(err) + fn parse_mac_args(&mut self) -> PResult<'a, P> { + self.parse_mac_args_common(true).map(P) + } + + fn parse_attr_args(&mut self) -> PResult<'a, MacArgs> { + self.parse_mac_args_common(false) + } + + fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, MacArgs> { + Ok(if self.check(&token::OpenDelim(DelimToken::Paren)) || + self.check(&token::OpenDelim(DelimToken::Bracket)) || + self.check(&token::OpenDelim(DelimToken::Brace)) { + match self.parse_token_tree() { + TokenTree::Delimited(dspan, delim, tokens) => + // We've confirmed above that there is a delimiter so unwrapping is OK. + MacArgs::Delimited(dspan, MacDelimiter::from_token(delim).unwrap(), tokens), + _ => unreachable!(), } - }; - let tts = match self.parse_token_tree() { - TokenTree::Delimited(_, _, tts) => tts, - _ => unreachable!(), - }; - let delim = match delim { - token::Paren => MacDelimiter::Parenthesis, - token::Bracket => MacDelimiter::Bracket, - token::Brace => MacDelimiter::Brace, - token::NoDelim => self.bug("unexpected no delimiter"), - }; - Ok((delim, tts.into())) + } else if !delimited_only { + if self.eat(&token::Eq) { + let eq_span = self.prev_span; + let mut is_interpolated_expr = false; + if let token::Interpolated(nt) = &self.token.kind { + if let token::NtExpr(..) = **nt { + is_interpolated_expr = true; + } + } + let token_tree = if is_interpolated_expr { + // We need to accept arbitrary interpolated expressions to continue + // supporting things like `doc = $expr` that work on stable. + // Non-literal interpolated expressions are rejected after expansion. + self.parse_token_tree() + } else { + self.parse_unsuffixed_lit()?.token_tree() + }; + + MacArgs::Eq(eq_span, token_tree.into()) + } else { + MacArgs::Empty + } + } else { + return self.unexpected(); + }) } fn parse_or_use_outer_attributes( diff --git a/src/librustc_parse/parser/pat.rs b/src/librustc_parse/parser/pat.rs index b068a4f16a533..1127c4b2d5f88 100644 --- a/src/librustc_parse/parser/pat.rs +++ b/src/librustc_parse/parser/pat.rs @@ -338,7 +338,7 @@ impl<'a> Parser<'a> { (None, self.parse_path(PathStyle::Expr)?) }; match self.token.kind { - token::Not if qself.is_none() => self.parse_pat_mac_invoc(lo, path)?, + token::Not if qself.is_none() => self.parse_pat_mac_invoc(path)?, token::DotDotDot | token::DotDotEq | token::DotDot => { self.parse_pat_range_starting_with_path(lo, qself, path)? } @@ -593,14 +593,12 @@ impl<'a> Parser<'a> { } /// Parse macro invocation - fn parse_pat_mac_invoc(&mut self, lo: Span, path: Path) -> PResult<'a, PatKind> { + fn parse_pat_mac_invoc(&mut self, path: Path) -> PResult<'a, PatKind> { self.bump(); - let (delim, tts) = self.expect_delimited_token_tree()?; + let args = self.parse_mac_args()?; let mac = Mac { path, - tts, - delim, - span: lo.to(self.prev_span), + args, prior_type_ascription: self.last_type_ascription, }; Ok(PatKind::Mac(mac)) diff --git a/src/librustc_parse/parser/path.rs b/src/librustc_parse/parser/path.rs index 68307440712a8..75bb67d47bc48 100644 --- a/src/librustc_parse/parser/path.rs +++ b/src/librustc_parse/parser/path.rs @@ -2,6 +2,7 @@ use super::{Parser, TokenType}; use crate::maybe_whole; use syntax::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs}; use syntax::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode}; +use syntax::ast::MacArgs; use syntax::ThinVec; use syntax::token::{self, Token}; use syntax::source_map::{Span, BytePos}; @@ -114,9 +115,9 @@ impl<'a> Parser<'a> { fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> { let meta_ident = match self.token.kind { token::Interpolated(ref nt) => match **nt { - token::NtMeta(ref item) => match item.tokens.is_empty() { - true => Some(item.path.clone()), - false => None, + token::NtMeta(ref item) => match item.args { + MacArgs::Empty => Some(item.path.clone()), + _ => None, }, _ => None, }, diff --git a/src/librustc_parse/parser/stmt.rs b/src/librustc_parse/parser/stmt.rs index a5f20691d077c..b952e8814a361 100644 --- a/src/librustc_parse/parser/stmt.rs +++ b/src/librustc_parse/parser/stmt.rs @@ -10,7 +10,7 @@ use syntax::ThinVec; use syntax::ptr::P; use syntax::ast; use syntax::ast::{DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; -use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; +use syntax::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac}; use syntax::util::classify; use syntax::token; use syntax::source_map::{respan, Span}; @@ -93,10 +93,11 @@ impl<'a> Parser<'a> { })); } - let (delim, tts) = self.expect_delimited_token_tree()?; + let args = self.parse_mac_args()?; + let delim = args.delim(); let hi = self.prev_span; - let style = if delim == MacDelimiter::Brace { + let style = if delim == token::Brace { MacStmtStyle::Braces } else { MacStmtStyle::NoBraces @@ -104,12 +105,10 @@ impl<'a> Parser<'a> { let mac = Mac { path, - tts, - delim, - span: lo.to(hi), + args, prior_type_ascription: self.last_type_ascription, }; - let kind = if delim == MacDelimiter::Brace || + let kind = if delim == token::Brace || self.token == token::Semi || self.token == token::Eof { StmtKind::Mac(P((mac, style, attrs.into()))) } @@ -130,7 +129,7 @@ impl<'a> Parser<'a> { self.warn_missing_semicolon(); StmtKind::Mac(P((mac, style, attrs.into()))) } else { - let e = self.mk_expr(mac.span, ExprKind::Mac(mac), ThinVec::new()); + let e = self.mk_expr(lo.to(hi), ExprKind::Mac(mac), ThinVec::new()); let e = self.maybe_recover_from_bad_qpath(e, true)?; let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; diff --git a/src/librustc_parse/parser/ty.rs b/src/librustc_parse/parser/ty.rs index 8e6bc29be5218..321427969051c 100644 --- a/src/librustc_parse/parser/ty.rs +++ b/src/librustc_parse/parser/ty.rs @@ -177,12 +177,10 @@ impl<'a> Parser<'a> { let path = self.parse_path(PathStyle::Type)?; if self.eat(&token::Not) { // Macro invocation in type position - let (delim, tts) = self.expect_delimited_token_tree()?; + let args = self.parse_mac_args()?; let mac = Mac { path, - tts, - delim, - span: lo.to(self.prev_span), + args, prior_type_ascription: self.last_type_ascription, }; TyKind::Mac(mac) diff --git a/src/librustc_parse/validate_attr.rs b/src/librustc_parse/validate_attr.rs index a3c9e2665930d..0fb348efece58 100644 --- a/src/librustc_parse/validate_attr.rs +++ b/src/librustc_parse/validate_attr.rs @@ -2,11 +2,9 @@ use errors::{PResult, Applicability}; use rustc_feature::{AttributeTemplate, BUILTIN_ATTRIBUTE_MAP}; -use syntax::ast::{self, Attribute, AttrKind, Ident, MetaItem, MetaItemKind}; +use syntax::ast::{self, Attribute, AttrKind, Ident, MacArgs, MetaItem, MetaItemKind}; use syntax::attr::mk_name_value_item_str; use syntax::early_buffered_lints::BufferedEarlyLintId; -use syntax::token; -use syntax::tokenstream::TokenTree; use syntax::sess::ParseSess; use syntax_pos::{Symbol, sym}; @@ -19,11 +17,9 @@ pub fn check_meta(sess: &ParseSess, attr: &Attribute) { // `rustc_dummy` doesn't have any restrictions specific to built-in attributes. Some((name, _, template, _)) if name != sym::rustc_dummy => check_builtin_attribute(sess, attr, name, template), - _ => if let Some(TokenTree::Token(token)) = attr.get_normal_item().tokens.trees().next() { - if token == token::Eq { - // All key-value attributes are restricted to meta-item syntax. - parse_meta(sess, attr).map_err(|mut err| err.emit()).ok(); - } + _ => if let MacArgs::Eq(..) = attr.get_normal_item().args { + // All key-value attributes are restricted to meta-item syntax. + parse_meta(sess, attr).map_err(|mut err| err.emit()).ok(); } } } diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs index 5a29a56ad5472..29cfee8408f30 100644 --- a/src/librustc_passes/ast_validation.rs +++ b/src/librustc_passes/ast_validation.rs @@ -737,14 +737,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> { |this| visit::walk_enum_def(this, enum_definition, generics, item_id)) } - fn visit_mac(&mut self, mac: &Mac) { - // when a new macro kind is added but the author forgets to set it up for expansion - // because that's the only part that won't cause a compiler error - self.session.diagnostic() - .span_bug(mac.span, "macro invocation missed in expansion; did you forget to override \ - the relevant `fold_*()` method in `PlaceholderExpander`?"); - } - fn visit_impl_item(&mut self, ii: &'a ImplItem) { if let ImplItemKind::Method(ref sig, _) = ii.kind { self.check_fn_decl(&sig.decl); diff --git a/src/librustc_save_analysis/dump_visitor.rs b/src/librustc_save_analysis/dump_visitor.rs index 5bec5b5eb6bfd..396d948433961 100644 --- a/src/librustc_save_analysis/dump_visitor.rs +++ b/src/librustc_save_analysis/dump_visitor.rs @@ -1515,14 +1515,6 @@ impl<'l, 'tcx> Visitor<'l> for DumpVisitor<'l, 'tcx> { } } - fn visit_mac(&mut self, mac: &'l ast::Mac) { - // These shouldn't exist in the AST at this point, log a span bug. - span_bug!( - mac.span, - "macro invocation should have been expanded out of AST" - ); - } - fn visit_pat(&mut self, p: &'l ast::Pat) { self.process_macro_use(p.span); self.process_pat(p); diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index 4b5fc7c2a1e54..7ee1054dc4846 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -482,7 +482,7 @@ fn build_macro(cx: &DocContext<'_>, did: DefId, name: ast::Name) -> clean::ItemE match cx.enter_resolver(|r| r.cstore().load_macro_untracked(did, cx.sess())) { LoadedMacro::MacroDef(def, _) => { let matchers: hir::HirVec = if let ast::ItemKind::MacroDef(ref def) = def.kind { - let tts: Vec<_> = def.stream().into_trees().collect(); + let tts: Vec<_> = def.body.inner_tokens().into_trees().collect(); tts.chunks(4).map(|arm| arm[0].span()).collect() } else { unreachable!() diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 512f43c86ca01..8018e005b12d7 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -27,7 +27,7 @@ pub use syntax_pos::symbol::{Ident, Symbol as Name}; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; use crate::token::{self, DelimToken}; -use crate::tokenstream::TokenStream; +use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; use syntax_pos::symbol::{kw, sym, Symbol}; use syntax_pos::{Span, DUMMY_SP, ExpnId}; @@ -40,6 +40,7 @@ use rustc_index::vec::Idx; use rustc_serialize::{self, Decoder, Encoder}; use rustc_macros::HashStable_Generic; +use std::iter; use std::fmt; #[cfg(test)] @@ -1372,34 +1373,89 @@ pub enum Movability { Movable, } -/// Represents a macro invocation. The `Path` indicates which macro -/// is being invoked, and the vector of token-trees contains the source -/// of the macro invocation. -/// -/// N.B., the additional ident for a `macro_rules`-style macro is actually -/// stored in the enclosing item. +/// Represents a macro invocation. The `path` indicates which macro +/// is being invoked, and the `args` are arguments passed to it. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Mac { pub path: Path, - pub delim: MacDelimiter, - pub tts: TokenStream, - pub span: Span, + pub args: P, pub prior_type_ascription: Option<(Span, bool)>, } -#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)] +impl Mac { + pub fn span(&self) -> Span { + self.path.span.to(self.args.span().unwrap_or(self.path.span)) + } +} + +/// Arguments passed to an attribute or a function-like macro. +#[derive(Clone, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] +pub enum MacArgs { + /// No arguments - `#[attr]`. + Empty, + /// Delimited arguments - `#[attr()/[]/{}]` or `mac!()/[]/{}`. + Delimited(DelimSpan, MacDelimiter, TokenStream), + /// Arguments of a key-value attribute - `#[attr = "value"]`. + Eq( + /// Span of the `=` token. + Span, + /// Token stream of the "value". + TokenStream, + ), +} + +impl MacArgs { + pub fn delim(&self) -> DelimToken { + match self { + MacArgs::Delimited(_, delim, _) => delim.to_token(), + MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim, + } + } + + pub fn span(&self) -> Option { + match *self { + MacArgs::Empty => None, + MacArgs::Delimited(dspan, ..) => Some(dspan.entire()), + MacArgs::Eq(eq_span, ref tokens) => Some(eq_span.to(tokens.span().unwrap_or(eq_span))), + } + } + + /// Tokens inside the delimiters or after `=`. + /// Proc macros see these tokens, for example. + pub fn inner_tokens(&self) -> TokenStream { + match self { + MacArgs::Empty => TokenStream::default(), + MacArgs::Delimited(.., tokens) | + MacArgs::Eq(.., tokens) => tokens.clone(), + } + } + + /// Tokens together with the delimiters or `=`. + /// Use of this method generally means that something suboptimal or hacky is happening. + pub fn outer_tokens(&self) -> TokenStream { + match *self { + MacArgs::Empty => TokenStream::default(), + MacArgs::Delimited(dspan, delim, ref tokens) => + TokenTree::Delimited(dspan, delim.to_token(), tokens.clone()).into(), + MacArgs::Eq(eq_span, ref tokens) => iter::once(TokenTree::token(token::Eq, eq_span)) + .chain(tokens.trees()).collect(), + } + } + + /// Whether a macro with these arguments needs a semicolon + /// when used as a standalone item or statement. + pub fn need_semicolon(&self) -> bool { + !matches!(self, MacArgs::Delimited(_, MacDelimiter::Brace ,_)) + } +} + +#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] pub enum MacDelimiter { Parenthesis, Bracket, Brace, } -impl Mac { - pub fn stream(&self) -> TokenStream { - self.tts.clone() - } -} - impl MacDelimiter { crate fn to_token(self) -> DelimToken { match self { @@ -1408,22 +1464,25 @@ impl MacDelimiter { MacDelimiter::Brace => DelimToken::Brace, } } + + pub fn from_token(delim: DelimToken) -> Option { + match delim { + token::Paren => Some(MacDelimiter::Parenthesis), + token::Bracket => Some(MacDelimiter::Bracket), + token::Brace => Some(MacDelimiter::Brace), + token::NoDelim => None, + } + } } /// Represents a macro definition. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct MacroDef { - pub tokens: TokenStream, + pub body: P, /// `true` if macro was defined with `macro_rules`. pub legacy: bool, } -impl MacroDef { - pub fn stream(&self) -> TokenStream { - self.tokens.clone().into() - } -} - // Clippy uses Hash and PartialEq #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq, HashStable_Generic)] pub enum StrStyle { @@ -2323,7 +2382,7 @@ impl rustc_serialize::Decodable for AttrId { #[derive(Clone, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] pub struct AttrItem { pub path: Path, - pub tokens: TokenStream, + pub args: MacArgs, } /// Metadata associated with an item. diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 29eff5c298169..079a0f6fafa2c 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -10,7 +10,7 @@ pub use crate::ast::Attribute; use crate::ast; use crate::ast::{AttrItem, AttrId, AttrKind, AttrStyle, Name, Ident, Path, PathSegment}; -use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem}; +use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem}; use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam}; use crate::mut_visit::visit_clobber; use crate::source_map::{BytePos, Spanned}; @@ -198,7 +198,7 @@ impl Attribute { pub fn is_word(&self) -> bool { if let AttrKind::Normal(item) = &self.kind { - item.tokens.is_empty() + matches!(item.args, MacArgs::Empty) } else { false } @@ -278,17 +278,9 @@ impl MetaItem { impl AttrItem { pub fn meta(&self, span: Span) -> Option { - let mut tokens = self.tokens.trees().peekable(); Some(MetaItem { path: self.path.clone(), - kind: if let Some(kind) = MetaItemKind::from_tokens(&mut tokens) { - if tokens.peek().is_some() { - return None; - } - kind - } else { - return None; - }, + kind: MetaItemKind::from_mac_args(&self.args)?, span, }) } @@ -362,8 +354,8 @@ crate fn mk_attr_id() -> AttrId { AttrId(id) } -pub fn mk_attr(style: AttrStyle, path: Path, tokens: TokenStream, span: Span) -> Attribute { - mk_attr_from_item(style, AttrItem { path, tokens }, span) +pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute { + mk_attr_from_item(style, AttrItem { path, args }, span) } pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute { @@ -377,12 +369,12 @@ pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attrib /// Returns an inner attribute with the given value and span. pub fn mk_attr_inner(item: MetaItem) -> Attribute { - mk_attr(AttrStyle::Inner, item.path, item.kind.tokens(item.span), item.span) + mk_attr(AttrStyle::Inner, item.path, item.kind.mac_args(item.span), item.span) } /// Returns an outer attribute with the given value and span. pub fn mk_attr_outer(item: MetaItem) -> Attribute { - mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span) + mk_attr(AttrStyle::Outer, item.path, item.kind.mac_args(item.span), item.span) } pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute { @@ -520,7 +512,26 @@ impl MetaItem { } impl MetaItemKind { - pub fn token_trees_and_joints(&self, span: Span) -> Vec { + pub fn mac_args(&self, span: Span) -> MacArgs { + match self { + MetaItemKind::Word => MacArgs::Empty, + MetaItemKind::NameValue(lit) => MacArgs::Eq(span, lit.token_tree().into()), + MetaItemKind::List(list) => { + let mut tts = Vec::new(); + for (i, item) in list.iter().enumerate() { + if i > 0 { + tts.push(TokenTree::token(token::Comma, span).into()); + } + tts.extend(item.token_trees_and_joints()) + } + MacArgs::Delimited( + DelimSpan::from_single(span), MacDelimiter::Parenthesis, TokenStream::new(tts) + ) + } + } + } + + fn token_trees_and_joints(&self, span: Span) -> Vec { match *self { MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { @@ -548,33 +559,8 @@ impl MetaItemKind { } } - // Premature conversions of `TokenTree`s to `TokenStream`s can hurt - // performance. Do not use this function if `token_trees_and_joints()` can - // be used instead. - pub fn tokens(&self, span: Span) -> TokenStream { - TokenStream::new(self.token_trees_and_joints(span)) - } - - fn from_tokens(tokens: &mut iter::Peekable) -> Option - where I: Iterator, - { - let delimited = match tokens.peek().cloned() { - Some(TokenTree::Token(token)) if token == token::Eq => { - tokens.next(); - return if let Some(TokenTree::Token(token)) = tokens.next() { - Lit::from_token(&token).ok().map(MetaItemKind::NameValue) - } else { - None - }; - } - Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => { - tokens.next(); - tts.clone() - } - _ => return Some(MetaItemKind::Word), - }; - - let mut tokens = delimited.into_trees().peekable(); + fn list_from_tokens(tokens: TokenStream) -> Option { + let mut tokens = tokens.into_trees().peekable(); let mut result = Vec::new(); while let Some(..) = tokens.peek() { let item = NestedMetaItem::from_tokens(&mut tokens)?; @@ -586,6 +572,47 @@ impl MetaItemKind { } Some(MetaItemKind::List(result)) } + + fn name_value_from_tokens( + tokens: &mut impl Iterator, + ) -> Option { + match tokens.next() { + Some(TokenTree::Token(token)) => + Lit::from_token(&token).ok().map(MetaItemKind::NameValue), + _ => None, + } + } + + fn from_mac_args(args: &MacArgs) -> Option { + match args { + MacArgs::Delimited(_, MacDelimiter::Parenthesis, tokens) => + MetaItemKind::list_from_tokens(tokens.clone()), + MacArgs::Delimited(..) => None, + MacArgs::Eq(_, tokens) => { + assert!(tokens.len() == 1); + MetaItemKind::name_value_from_tokens(&mut tokens.trees()) + } + MacArgs::Empty => Some(MetaItemKind::Word), + } + } + + fn from_tokens( + tokens: &mut iter::Peekable>, + ) -> Option { + match tokens.peek() { + Some(TokenTree::Delimited(_, token::Paren, inner_tokens)) => { + let inner_tokens = inner_tokens.clone(); + tokens.next(); + MetaItemKind::list_from_tokens(inner_tokens) + } + Some(TokenTree::Delimited(..)) => None, + Some(TokenTree::Token(Token { kind: token::Eq, .. })) => { + tokens.next(); + MetaItemKind::name_value_from_tokens(tokens) + } + _ => Some(MetaItemKind::Word), + } + } } impl NestedMetaItem { diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 3d4a5d624c119..3dcdd4db6377a 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -12,6 +12,7 @@ #![feature(const_transmute)] #![feature(crate_visibility_modifier)] #![feature(label_break_value)] +#![feature(matches_macro)] #![feature(nll)] #![feature(try_trait)] #![feature(slice_patterns)] diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index fbe28215a56c8..8889e5df26c52 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -359,6 +359,26 @@ pub fn visit_fn_sig(FnSig { header, decl }: &mut FnSig, vis: &mut vis.visit_fn_decl(decl); } +// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. +pub fn visit_mac_args(args: &mut MacArgs, vis: &mut T) { + match args { + MacArgs::Empty => {} + MacArgs::Delimited(dspan, _delim, tokens) => { + visit_delim_span(dspan, vis); + vis.visit_tts(tokens); + } + MacArgs::Eq(eq_span, tokens) => { + vis.visit_span(eq_span); + vis.visit_tts(tokens); + } + } +} + +pub fn visit_delim_span(dspan: &mut DelimSpan, vis: &mut T) { + vis.visit_span(&mut dspan.open); + vis.visit_span(&mut dspan.close); +} + pub fn noop_flat_map_field_pattern( mut fp: FieldPat, vis: &mut T, @@ -550,9 +570,9 @@ pub fn noop_visit_local(local: &mut P, vis: &mut T) { pub fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { let Attribute { kind, id: _, style: _, span } = attr; match kind { - AttrKind::Normal(AttrItem { path, tokens }) => { + AttrKind::Normal(AttrItem { path, args }) => { vis.visit_path(path); - vis.visit_tts(tokens); + visit_mac_args(args, vis); } AttrKind::DocComment(_) => {} } @@ -560,15 +580,14 @@ pub fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { } pub fn noop_visit_mac(mac: &mut Mac, vis: &mut T) { - let Mac { path, delim: _, tts, span, prior_type_ascription: _ } = mac; + let Mac { path, args, prior_type_ascription: _ } = mac; vis.visit_path(path); - vis.visit_tts(tts); - vis.visit_span(span); + visit_mac_args(args, vis); } pub fn noop_visit_macro_def(macro_def: &mut MacroDef, vis: &mut T) { - let MacroDef { tokens, legacy: _ } = macro_def; - vis.visit_tts(tokens); + let MacroDef { body, legacy: _ } = macro_def; + visit_mac_args(body, vis); } pub fn noop_visit_meta_list_item(li: &mut NestedMetaItem, vis: &mut T) { @@ -682,9 +701,9 @@ pub fn noop_visit_interpolated(nt: &mut token::Nonterminal, vis: token::NtIdent(ident, _is_raw) => vis.visit_ident(ident), token::NtLifetime(ident) => vis.visit_ident(ident), token::NtLiteral(expr) => vis.visit_expr(expr), - token::NtMeta(AttrItem { path, tokens }) => { + token::NtMeta(AttrItem { path, args }) => { vis.visit_path(path); - vis.visit_tts(tokens); + visit_mac_args(args, vis); } token::NtPath(path) => vis.visit_path(path), token::NtTT(tt) => vis.visit_tt(tt), diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 0d2e8dddce671..4821bbd9ec6e2 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1,6 +1,6 @@ use crate::ast::{self, BlockCheckMode, PatKind, RangeEnd, RangeSyntax}; use crate::ast::{SelfKind, GenericBound, TraitBoundModifier}; -use crate::ast::{Attribute, MacDelimiter, GenericArg}; +use crate::ast::{Attribute, GenericArg, MacArgs}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::util::comments; use crate::attr; @@ -639,17 +639,22 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere fn print_attr_item(&mut self, item: &ast::AttrItem, span: Span) { self.ibox(0); - match item.tokens.trees().next() { - Some(TokenTree::Delimited(_, delim, tts)) => { - self.print_mac_common( - Some(MacHeader::Path(&item.path)), false, None, delim, tts, true, span - ); - } - tree => { + match &item.args { + MacArgs::Delimited(_, delim, tokens) => self.print_mac_common( + Some(MacHeader::Path(&item.path)), + false, + None, + delim.to_token(), + tokens.clone(), + true, + span, + ), + MacArgs::Empty | MacArgs::Eq(..) => { self.print_path(&item.path, false, 0); - if tree.is_some() { + if let MacArgs::Eq(_, tokens) = &item.args { self.space(); - self.print_tts(item.tokens.clone(), true); + self.word_space("="); + self.print_tts(tokens.clone(), true); } } } @@ -1097,9 +1102,8 @@ impl<'a> State<'a> { } ast::ForeignItemKind::Macro(ref m) => { self.print_mac(m); - match m.delim { - MacDelimiter::Brace => {}, - _ => self.s.word(";") + if m.args.need_semicolon() { + self.s.word(";"); } } } @@ -1361,9 +1365,8 @@ impl<'a> State<'a> { } ast::ItemKind::Mac(ref mac) => { self.print_mac(mac); - match mac.delim { - MacDelimiter::Brace => {} - _ => self.s.word(";"), + if mac.args.need_semicolon() { + self.s.word(";"); } } ast::ItemKind::MacroDef(ref macro_def) => { @@ -1377,8 +1380,8 @@ impl<'a> State<'a> { Some(MacHeader::Keyword(kw)), has_bang, Some(item.ident), - DelimToken::Brace, - macro_def.stream(), + macro_def.body.delim(), + macro_def.body.inner_tokens(), true, item.span, ); @@ -1578,9 +1581,8 @@ impl<'a> State<'a> { } ast::TraitItemKind::Macro(ref mac) => { self.print_mac(mac); - match mac.delim { - MacDelimiter::Brace => {} - _ => self.s.word(";"), + if mac.args.need_semicolon() { + self.s.word(";"); } } } @@ -1608,9 +1610,8 @@ impl<'a> State<'a> { } ast::ImplItemKind::Macro(ref mac) => { self.print_mac(mac); - match mac.delim { - MacDelimiter::Brace => {} - _ => self.s.word(";"), + if mac.args.need_semicolon() { + self.s.word(";"); } } } @@ -1775,10 +1776,10 @@ impl<'a> State<'a> { Some(MacHeader::Path(&m.path)), true, None, - m.delim.to_token(), - m.stream(), + m.args.delim(), + m.args.inner_tokens(), true, - m.span, + m.span(), ); } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 6a0523dd655b8..491b9a9ade47a 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -225,6 +225,14 @@ impl TokenStream { self.0.len() } + pub fn span(&self) -> Option { + match &**self.0 { + [] => None, + [(tt, _)] => Some(tt.span()), + [(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())), + } + } + pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { match streams.len() { 0 => TokenStream::default(), diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 5ff337fb60e28..4ee09b4b87afa 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -841,11 +841,19 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) { pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) { match attr.kind { - AttrKind::Normal(ref item) => visitor.visit_tts(item.tokens.clone()), + AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args), AttrKind::DocComment(_) => {} } } +pub fn walk_mac_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a MacArgs) { + match args { + MacArgs::Empty => {} + MacArgs::Delimited(_dspan, _delim, tokens) => visitor.visit_tts(tokens.clone()), + MacArgs::Eq(_eq_span, tokens) => visitor.visit_tts(tokens.clone()), + } +} + pub fn walk_tt<'a, V: Visitor<'a>>(visitor: &mut V, tt: TokenTree) { match tt { TokenTree::Token(token) => visitor.visit_token(token), diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs index a6ced1439c5d9..9bfedb3b6174e 100644 --- a/src/libsyntax_expand/expand.rs +++ b/src/libsyntax_expand/expand.rs @@ -11,7 +11,7 @@ use rustc_parse::DirectoryOwnership; use rustc_parse::parser::Parser; use rustc_parse::validate_attr; use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; -use syntax::ast::{MacStmtStyle, StmtKind, ItemKind}; +use syntax::ast::{MacArgs, MacStmtStyle, StmtKind, ItemKind}; use syntax::attr::{self, HasAttrs, is_builtin_attr}; use syntax::source_map::respan; use syntax::feature_gate::{self, feature_err}; @@ -597,13 +597,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> { InvocationKind::Bang { mac, .. } => match ext { SyntaxExtensionKind::Bang(expander) => { self.gate_proc_macro_expansion_kind(span, fragment_kind); - let tok_result = expander.expand(self.cx, span, mac.stream()); + let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) } SyntaxExtensionKind::LegacyBang(expander) => { let prev = self.cx.current_expansion.prior_type_ascription; self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; - let tok_result = expander.expand(self.cx, span, mac.stream()); + let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); let result = if let Some(result) = fragment_kind.make_from(tok_result) { result } else { @@ -642,8 +642,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { => panic!("unexpected annotatable"), })), DUMMY_SP).into(); let item = attr.unwrap_normal_item(); - let input = self.extract_proc_macro_attr_input(item.tokens, span); - let tok_result = expander.expand(self.cx, span, input, item_tok); + if let MacArgs::Eq(..) = item.args { + self.cx.span_err(span, "key-value macro attributes are not supported"); + } + let tok_result = + expander.expand(self.cx, span, item.args.inner_tokens(), item_tok); self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span) } SyntaxExtensionKind::LegacyAttr(expander) => { @@ -687,23 +690,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } } - fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { - let mut trees = tokens.trees(); - match trees.next() { - Some(TokenTree::Delimited(_, _, tts)) => { - if trees.next().is_none() { - return tts.into() - } - } - Some(TokenTree::Token(..)) => {} - None => return TokenStream::default(), - } - self.cx.span_err(span, "custom attribute invocations must be \ - of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ - followed by a delimiter token"); - TokenStream::default() - } - fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { let kind = match item { Annotatable::Item(item) => match &item.kind { @@ -1560,7 +1546,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); *at = attr::Attribute { kind: ast::AttrKind::Normal( - AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span) }, ), span: at.span, id: at.id, diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs index b191527df1991..e3c3655bcf882 100644 --- a/src/libsyntax_expand/mbe/macro_rules.rs +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -318,8 +318,8 @@ pub fn compile_declarative_macro( let tt_spec = ast::Ident::new(sym::tt, def.span); // Parse the macro_rules! invocation - let body = match def.kind { - ast::ItemKind::MacroDef(ref body) => body, + let (is_legacy, body) = match &def.kind { + ast::ItemKind::MacroDef(macro_def) => (macro_def.legacy, macro_def.body.inner_tokens()), _ => unreachable!(), }; @@ -338,7 +338,7 @@ pub fn compile_declarative_macro( mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), ], separator: Some(Token::new( - if body.legacy { token::Semi } else { token::Comma }, + if is_legacy { token::Semi } else { token::Comma }, def.span, )), kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), @@ -350,7 +350,7 @@ pub fn compile_declarative_macro( DelimSpan::dummy(), Lrc::new(mbe::SequenceRepetition { tts: vec![mbe::TokenTree::token( - if body.legacy { token::Semi } else { token::Comma }, + if is_legacy { token::Semi } else { token::Comma }, def.span, )], separator: None, @@ -360,7 +360,7 @@ pub fn compile_declarative_macro( ), ]; - let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { + let argument_map = match parse(sess, body, &argument_gram, None, true) { Success(m) => m, Failure(token, msg) => { let s = parse_failure_msg(&token); @@ -435,7 +435,7 @@ pub fn compile_declarative_macro( // that is not lint-checked and trigger the "failed to process buffered lint here" bug. valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); - let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); + let (transparency, transparency_error) = attr::find_transparency(&def.attrs, is_legacy); match transparency_error { Some(TransparencyError::UnknownTransparency(value, span)) => diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs index 4092d4b97de04..a1157667df1b4 100644 --- a/src/libsyntax_expand/mbe/transcribe.rs +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -30,13 +30,6 @@ impl MutVisitor for Marker { } } -impl Marker { - fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { - self.visit_span(&mut dspan.open); - self.visit_span(&mut dspan.close); - } -} - /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). enum Frame { Delimited { forest: Lrc, idx: usize, span: DelimSpan }, @@ -271,7 +264,7 @@ pub(super) fn transcribe( // jump back out of the Delimited, pop the result_stack and add the new results back to // the previous results (from outside the Delimited). mbe::TokenTree::Delimited(mut span, delimited) => { - marker.visit_delim_span(&mut span); + mut_visit::visit_delim_span(&mut span, &mut marker); stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); result_stack.push(mem::take(&mut result)); } diff --git a/src/libsyntax_expand/parse/tests.rs b/src/libsyntax_expand/parse/tests.rs index 08950ddefbaee..30e83c151e255 100644 --- a/src/libsyntax_expand/parse/tests.rs +++ b/src/libsyntax_expand/parse/tests.rs @@ -272,7 +272,7 @@ fn ttdelim_span() { "foo!( fn main() { body } )".to_string(), &sess).unwrap(); let tts: Vec<_> = match expr.kind { - ast::ExprKind::Mac(ref mac) => mac.stream().trees().collect(), + ast::ExprKind::Mac(ref mac) => mac.args.inner_tokens().trees().collect(), _ => panic!("not a macro"), }; diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs index 6cbe8c132457c..74ade1de20e2a 100644 --- a/src/libsyntax_expand/placeholders.rs +++ b/src/libsyntax_expand/placeholders.rs @@ -3,7 +3,6 @@ use crate::expand::{AstFragment, AstFragmentKind}; use syntax::ast; use syntax::source_map::{DUMMY_SP, dummy_spanned}; -use syntax::tokenstream::TokenStream; use syntax::mut_visit::*; use syntax::ptr::P; use syntax::ThinVec; @@ -17,9 +16,7 @@ pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId, vis: Option ast::Mac { ast::Mac { path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - tts: TokenStream::default().into(), - delim: ast::MacDelimiter::Brace, - span: DUMMY_SP, + args: P(ast::MacArgs::Empty), prior_type_ascription: None, } } diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs index 099cf0a4be904..8e56e2bb00b4b 100644 --- a/src/libsyntax_expand/proc_macro.rs +++ b/src/libsyntax_expand/proc_macro.rs @@ -1,7 +1,7 @@ use crate::base::{self, *}; use crate::proc_macro_server; -use syntax::ast::{self, ItemKind}; +use syntax::ast::{self, ItemKind, MacArgs}; use syntax::errors::{Applicability, FatalError}; use syntax::symbol::sym; use syntax::token; @@ -183,7 +183,7 @@ crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) } let parse_derive_paths = |attr: &ast::Attribute| { - if attr.get_normal_item().tokens.is_empty() { + if let MacArgs::Empty = attr.get_normal_item().args { return Ok(Vec::new()); } rustc_parse::parse_in_attr(cx.parse_sess, attr, |p| p.parse_derive_paths()) diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index c4f3c03813fcd..c788d06299405 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -6,7 +6,7 @@ use syntax::token::{self, TokenKind}; use syntax::print::pprust; use syntax::ptr::P; use syntax::symbol::{sym, Symbol}; -use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree}; use syntax_expand::base::*; use syntax_pos::{Span, DUMMY_SP}; @@ -26,19 +26,19 @@ pub fn expand_assert<'cx>( // `core::panic` and `std::panic` are different macros, so we use call-site // context to pick up whichever is currently in scope. let sp = cx.with_call_site_ctxt(sp); + let tokens = custom_message.unwrap_or_else(|| { + TokenStream::from(TokenTree::token( + TokenKind::lit(token::Str, Symbol::intern(&format!( + "assertion failed: {}", + pprust::expr_to_string(&cond_expr).escape_debug() + )), None), + DUMMY_SP, + )) + }); + let args = P(MacArgs::Delimited(DelimSpan::from_single(sp), MacDelimiter::Parenthesis, tokens)); let panic_call = Mac { path: Path::from_ident(Ident::new(sym::panic, sp)), - tts: custom_message.unwrap_or_else(|| { - TokenStream::from(TokenTree::token( - TokenKind::lit(token::Str, Symbol::intern(&format!( - "assertion failed: {}", - pprust::expr_to_string(&cond_expr).escape_debug() - )), None), - DUMMY_SP, - )) - }).into(), - delim: MacDelimiter::Parenthesis, - span: sp, + args, prior_type_ascription: None, }; let if_expr = cx.expr_if( diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs index 5c3009c432ca3..98cf8a34742e5 100644 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ b/src/libsyntax_ext/cmdline_attrs.rs @@ -16,7 +16,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - ); let start_span = parser.token.span; - let AttrItem { path, tokens } = panictry!(parser.parse_attr_item()); + let AttrItem { path, args } = panictry!(parser.parse_attr_item()); let end_span = parser.token.span; if parser.token != token::Eof { parse_sess.span_diagnostic @@ -24,7 +24,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - continue; } - krate.attrs.push(mk_attr(AttrStyle::Inner, path, tokens, start_span.to(end_span))); + krate.attrs.push(mk_attr(AttrStyle::Inner, path, args, start_span.to(end_span))); } krate diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs index b6bf2f881616f..5bd84b43a7801 100644 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ b/src/libsyntax_ext/deriving/generic/mod.rs @@ -340,14 +340,12 @@ pub fn combine_substructure(f: CombineSubstructureFunc<'_>) fn find_type_parameters( ty: &ast::Ty, ty_param_names: &[ast::Name], - span: Span, cx: &ExtCtxt<'_>, ) -> Vec> { use syntax::visit; struct Visitor<'a, 'b> { cx: &'a ExtCtxt<'b>, - span: Span, ty_param_names: &'a [ast::Name], types: Vec>, } @@ -366,18 +364,11 @@ fn find_type_parameters( } fn visit_mac(&mut self, mac: &ast::Mac) { - let span = mac.span.with_ctxt(self.span.ctxt()); - self.cx.span_err(span, "`derive` cannot be used on items with type macros"); + self.cx.span_err(mac.span(), "`derive` cannot be used on items with type macros"); } } - let mut visitor = Visitor { - ty_param_names, - types: Vec::new(), - span, - cx, - }; - + let mut visitor = Visitor { cx, ty_param_names, types: Vec::new() }; visit::Visitor::visit_ty(&mut visitor, ty); visitor.types @@ -605,7 +596,7 @@ impl<'a> TraitDef<'a> { .collect(); for field_ty in field_tys { - let tys = find_type_parameters(&field_ty, &ty_param_names, self.span, cx); + let tys = find_type_parameters(&field_ty, &ty_param_names, cx); for ty in tys { // if we have already handled this type, skip it diff --git a/src/test/ui/issues/issue-10536.rs b/src/test/ui/issues/issue-10536.rs index 111078abb3700..f536d8f940a93 100644 --- a/src/test/ui/issues/issue-10536.rs +++ b/src/test/ui/issues/issue-10536.rs @@ -11,9 +11,9 @@ macro_rules! foo{ pub fn main() { foo!(); - assert!({one! two()}); //~ ERROR expected open delimiter + assert!({one! two()}); //~ ERROR expected one of `(`, `[`, or `{`, found `two` // regardless of whether nested macro_rules works, the following should at // least throw a conventional error. - assert!({one! two}); //~ ERROR expected open delimiter + assert!({one! two}); //~ ERROR expected one of `(`, `[`, or `{`, found `two` } diff --git a/src/test/ui/issues/issue-10536.stderr b/src/test/ui/issues/issue-10536.stderr index 73f948107f185..cc048445871a4 100644 --- a/src/test/ui/issues/issue-10536.stderr +++ b/src/test/ui/issues/issue-10536.stderr @@ -1,14 +1,14 @@ -error: expected open delimiter +error: expected one of `(`, `[`, or `{`, found `two` --> $DIR/issue-10536.rs:14:19 | LL | assert!({one! two()}); - | ^^^ expected open delimiter + | ^^^ expected one of `(`, `[`, or `{` -error: expected open delimiter +error: expected one of `(`, `[`, or `{`, found `two` --> $DIR/issue-10536.rs:18:19 | LL | assert!({one! two}); - | ^^^ expected open delimiter + | ^^^ expected one of `(`, `[`, or `{` error: aborting due to 2 previous errors diff --git a/src/test/ui/parser/macro-bad-delimiter-ident.rs b/src/test/ui/parser/macro-bad-delimiter-ident.rs index 13dec95435be0..f461f06b4dca6 100644 --- a/src/test/ui/parser/macro-bad-delimiter-ident.rs +++ b/src/test/ui/parser/macro-bad-delimiter-ident.rs @@ -1,3 +1,3 @@ fn main() { - foo! bar < //~ ERROR expected open delimiter + foo! bar < //~ ERROR expected one of `(`, `[`, or `{`, found `bar` } diff --git a/src/test/ui/parser/macro-bad-delimiter-ident.stderr b/src/test/ui/parser/macro-bad-delimiter-ident.stderr index e97839a4f4a52..f2365fed273b1 100644 --- a/src/test/ui/parser/macro-bad-delimiter-ident.stderr +++ b/src/test/ui/parser/macro-bad-delimiter-ident.stderr @@ -1,8 +1,8 @@ -error: expected open delimiter +error: expected one of `(`, `[`, or `{`, found `bar` --> $DIR/macro-bad-delimiter-ident.rs:2:10 | LL | foo! bar < - | ^^^ expected open delimiter + | ^^^ expected one of `(`, `[`, or `{` error: aborting due to previous error diff --git a/src/test/ui/proc-macro/proc-macro-gates.rs b/src/test/ui/proc-macro/proc-macro-gates.rs index 0096a84f14c34..591c1e039bd75 100644 --- a/src/test/ui/proc-macro/proc-macro-gates.rs +++ b/src/test/ui/proc-macro/proc-macro-gates.rs @@ -18,7 +18,7 @@ mod _test2_inner { //~| ERROR: non-builtin inner attributes are unstable } -#[empty_attr = "y"] //~ ERROR: must only be followed by a delimiter token +#[empty_attr = "y"] //~ ERROR: key-value macro attributes are not supported fn _test3() {} fn attrs() { diff --git a/src/test/ui/proc-macro/proc-macro-gates.stderr b/src/test/ui/proc-macro/proc-macro-gates.stderr index 14a4f8c0fbca2..e939434243b6a 100644 --- a/src/test/ui/proc-macro/proc-macro-gates.stderr +++ b/src/test/ui/proc-macro/proc-macro-gates.stderr @@ -34,7 +34,7 @@ LL | #![empty_attr] = note: for more information, see https://github.com/rust-lang/rust/issues/54727 = help: add `#![feature(proc_macro_hygiene)]` to the crate attributes to enable -error: custom attribute invocations must be of the form `#[foo]` or `#[foo(..)]`, the macro name must only be followed by a delimiter token +error: key-value macro attributes are not supported --> $DIR/proc-macro-gates.rs:21:1 | LL | #[empty_attr = "y"]