Skip to content

Commit

Permalink
Remove NtBlock.
Browse files Browse the repository at this point in the history 
XXX: also Nonterminal and TokenKind::Interpolated
  • Loading branch information
@nnethercote
nnethercote committed May 16, 2024
1 parent df017a6 commit 13fdc3d
Show file tree
Hide file tree
Showing 16 changed files with 98 additions and 356 deletions.
14 changes: 0 additions & 14 deletions compiler/rustc_ast/src/ast_traits.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
//! The traits are not implemented exhaustively, only when actually necessary.

use crate::ptr::P;
use crate::token::Nonterminal;
use crate::tokenstream::LazyAttrTokenStream;
use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
Expand Down Expand Up @@ -228,19 +227,6 @@ impl HasTokens for Attribute {
}
}

impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
Nonterminal::NtBlock(block) => block.tokens(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
Nonterminal::NtBlock(block) => block.tokens_mut(),
}
}
}

/// A trait for AST nodes having (or not having) attributes.
pub trait HasAttrs {
/// This is `true` if this `HasAttrs` might support 'custom' (proc-macro) inner
Expand Down
40 changes: 3 additions & 37 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -767,9 +767,9 @@ fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyAttrTokenStream>, vis
visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
}

/// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
/// In practice the ident part is not actually used by specific visitors right now,
/// but there's a test below checking that it works.
/// Applies ident visitor if it's an ident. In practice this is not actually
/// used by specific visitors right now, but there's a test below checking that
/// it works.
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
let Token { kind, span } = t;
Expand All @@ -787,45 +787,11 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
token::NtLifetime(ident) => {
vis.visit_ident(ident);
}
token::Interpolated(nt) => {
let nt = Lrc::make_mut(nt);
visit_nonterminal(nt, vis);
}
_ => {}
}
vis.visit_span(span);
}

// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
/// Applies the visitor to elements of interpolated nodes.
//
// N.B., this can occur only when applying a visitor to partially expanded
// code, where parsed pieces have gotten implanted ito *other* macro
// invocations. This is relevant for macro hygiene, but possibly not elsewhere.
//
// One problem here occurs because the types for flat_map_item, flat_map_stmt,
// etc., allow the visitor to return *multiple* items; this is a problem for the
// nodes here, because they insist on having exactly one piece. One solution
// would be to mangle the MutVisitor trait to include one-to-many and
// one-to-one versions of these entry points, but that would probably confuse a
// lot of people and help very few. Instead, I'm just going to put in dynamic
// checks. I think the performance impact of this will be pretty much
// nonexistent. The danger is that someone will apply a `MutVisitor` to a
// partially expanded node, and will be confused by the fact that their
// `flat_map_item` or `flat_map_stmt` isn't getting called on `NtItem` or `NtStmt`
// nodes. Hopefully they'll wind up reading this comment, and doing something
// appropriate.
//
// BTW, design choice: I considered just changing the type of, e.g., `NtItem` to
// contain multiple items, but decided against it when I looked at
// `parse_item_or_view_item` and tried to figure out what I would do with
// multiple items there....
fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
match nt {
token::NtBlock(block) => vis.visit_block(block),
}
}

// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_defaultness<T: MutVisitor>(defaultness: &mut Defaultness, vis: &mut T) {
match defaultness {
Expand Down
128 changes: 16 additions & 112 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
pub use BinOpToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use TokenKind::*;

use crate::ast;
use crate::ptr::P;
use crate::util::case::Case;

use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
use rustc_macros::{Decodable, Encodable, HashStable_Generic};
use rustc_span::symbol::{kw, sym};
#[allow(clippy::useless_attribute)] // FIXME: following use of `hidden_glob_reexports` incorrectly triggers `useless_attribute` lint.
Expand Down Expand Up @@ -282,9 +278,7 @@ impl From<IdentIsRaw> for bool {
}
}

// SAFETY: due to the `Clone` impl below, all fields of all variants other than
// `Interpolated` must impl `Copy`.
#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum TokenKind {
/* Expression-operator symbols. */
/// `=`
Expand Down Expand Up @@ -373,21 +367,6 @@ pub enum TokenKind {
/// the `lifetime` metavariable in the macro's RHS.
NtLifetime(Ident),

/// An embedded AST node, as produced by a macro. This only exists for
/// historical reasons. We'd like to get rid of it, for multiple reasons.
/// - It's conceptually very strange. Saying a token can contain an AST
/// node is like saying, in natural language, that a word can contain a
/// sentence.
/// - It requires special handling in a bunch of places in the parser.
/// - It prevents `Token` from implementing `Copy`.
/// It adds complexity and likely slows things down. Please don't add new
/// occurrences of this token kind!
///
/// The span in the surrounding `Token` is that of the metavariable in the
/// macro's RHS. The span within the Nonterminal is that of the fragment
/// passed to the macro at the call site.
Interpolated(Lrc<Nonterminal>),

/// A doc comment token.
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
/// similarly to symbols in string literal tokens.
Expand All @@ -397,19 +376,6 @@ pub enum TokenKind {
Eof,
}

impl Clone for TokenKind {
fn clone(&self) -> Self {
// `TokenKind` would impl `Copy` if it weren't for `Interpolated`. So
// for all other variants, this implementation of `clone` is just like
// a copy. This is faster than the `derive(Clone)` version which has a
// separate path for every variant.
match self {
Interpolated(nt) => Interpolated(nt.clone()),
_ => unsafe { std::ptr::read(self) },
}
}
}

#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct Token {
pub kind: TokenKind,
Expand Down Expand Up @@ -496,7 +462,6 @@ impl Token {
pub fn uninterpolated_span(&self) -> Span {
match self.kind {
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
Interpolated(ref nt) => nt.use_span(),
_ => self.span,
}
}
Expand All @@ -514,7 +479,7 @@ impl Token {
}

OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
}
}

Expand Down Expand Up @@ -542,7 +507,6 @@ impl Token {
PathSep | // global path
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Block |
NonterminalKind::Expr |
Expand All @@ -569,7 +533,6 @@ impl Token {
| DotDot | DotDotDot | DotDotEq // ranges
| Lt | BinOp(Shl) // associated path
| PathSep => true, // global path
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Block |
NonterminalKind::PatParam { .. } |
Expand Down Expand Up @@ -610,7 +573,6 @@ impl Token {
pub fn can_begin_const_arg(&self) -> bool {
match self.kind {
OpenDelim(Delimiter::Brace) => true,
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Expr | NonterminalKind::Block | NonterminalKind::Literal,
))) => true,
Expand Down Expand Up @@ -733,31 +695,20 @@ impl Token {
/// That is, is this a pre-parsed expression dropped into the token stream
/// (which happens while parsing the result of macro expansion)?
pub fn is_metavar_expr(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtBlock(_) = &**nt
{
true
} else if matches!(
matches!(
self.is_metavar_seq(),
Some(NonterminalKind::Expr | NonterminalKind::Literal | NonterminalKind::Path)
) {
true
} else {
false
}
Some(
NonterminalKind::Expr
| NonterminalKind::Literal
| NonterminalKind::Path
| NonterminalKind::Block
)
)
}

/// Is the token an interpolated block (`$b:block`)?
pub fn is_whole_block(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtBlock(..) = &**nt
{
return true;
}

false
/// Are we at a block from a metavar (`$b:block`)?
pub fn is_metavar_block(&self) -> bool {
matches!(self.is_metavar_seq(), Some(NonterminalKind::Block))
}

/// Returns `true` if the token is either the `mut` or `const` keyword.
Expand All @@ -782,7 +733,8 @@ impl Token {
self.is_non_raw_ident_where(|id| id.name == kw)
}

/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case.
/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this
/// token is an identifier equal to `kw` ignoring the case.
pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
self.is_keyword(kw)
|| (case == Case::Insensitive
Expand Down Expand Up @@ -903,7 +855,7 @@ impl Token {
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
| Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof => {
return None;
}
};
Expand All @@ -919,12 +871,6 @@ impl PartialEq<TokenKind> for Token {
}
}

#[derive(Clone, Encodable, Decodable)]
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtBlock(P<ast::Block>),
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NonterminalKind {
Item,
Expand Down Expand Up @@ -1003,47 +949,6 @@ impl fmt::Display for NonterminalKind {
}
}

impl Nonterminal {
pub fn use_span(&self) -> Span {
match self {
NtBlock(block) => block.span,
}
}

pub fn descr(&self) -> &'static str {
match self {
NtBlock(..) => "block",
}
}
}

impl PartialEq for Nonterminal {
fn eq(&self, _rhs: &Self) -> bool {
// FIXME: Assume that all nonterminals are not equal, we can't compare them
// correctly based on data from AST. This will prevent them from matching each other
// in macros. The comparison will become possible only when each nonterminal has an
// attached token stream from which it was parsed.
false
}
}

impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtBlock(..) => f.pad("NtBlock(..)"),
}
}
}

impl<CTX> HashStable<CTX> for Nonterminal
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
panic!("interpolated tokens should not be present in the HIR")
}
}

// Some types are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(target_pointer_width = "64")]
mod size_asserts {
Expand All @@ -1052,7 +957,6 @@ mod size_asserts {
// tidy-alphabetical-start
static_assert_size!(Lit, 12);
static_assert_size!(LitKind, 2);
static_assert_size!(Nonterminal, 8);
static_assert_size!(Token, 24);
static_assert_size!(TokenKind, 16);
// tidy-alphabetical-end
Expand Down
21 changes: 4 additions & 17 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

use crate::ast::AttrStyle;
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind};
use crate::AttrVec;

use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
Expand Down Expand Up @@ -474,12 +474,6 @@ impl TokenStream {
attr_stream.to_tokenstream()
}

pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
match nt {
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
}
}

fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
match token.kind {
token::NtIdent(ident, is_raw) => {
Expand All @@ -491,12 +485,6 @@ impl TokenStream {
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
),
token::Interpolated(ref nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
TokenStream::from_nonterminal_ast(&nt).flattened(),
),
_ => TokenTree::Token(token.clone(), spacing),
}
}
Expand All @@ -514,10 +502,9 @@ impl TokenStream {
pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(
token.kind,
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
),
TokenTree::Token(token, _) => {
!matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..))
}
TokenTree::Delimited(.., inner) => can_skip(inner),
})
}
Expand Down
6 changes: 1 addition & 5 deletions compiler/rustc_ast_pretty/src/pprust/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,11 @@ pub mod state;
pub use state::{print_crate, AnnNode, Comments, PpAnn, PrintState, State};

use rustc_ast as ast;
use rustc_ast::token::{Nonterminal, Token, TokenKind};
use rustc_ast::token::{Token, TokenKind};
use rustc_ast::tokenstream::{TokenStream, TokenTree};

use std::borrow::Cow;

pub fn nonterminal_to_string(nt: &Nonterminal) -> String {
State::new().nonterminal_to_string(nt)
}

/// Print the token kind precisely, without converting `$crate` into its respective crate name.
pub fn token_kind_to_string(tok: &TokenKind) -> Cow<'static, str> {
State::new().token_kind_to_string(tok)
Expand Down
Loading

0 comments on commit 13fdc3d

Please sign in to comment.