WIP

src/libsyntax/parse/lexer/tokentrees.rs

@@ -238,7 +238,7 @@ impl TokenStreamBuilder {
     fn push(&mut self, (tree, joint): TreeAndJoint) {
         if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
             if let TokenTree::Token(token) = &tree {
-                if let Some(glued) = prev_token.glue_for_parser(token) {
+                if let Some(glued) = prev_token.glue(token) {
                     self.buf.pop();
                     self.buf.push((TokenTree::Token(glued), joint));
                     return;

src/libsyntax/parse/parser.rs

@@ -24,8 +24,10 @@ use crate::ptr::P;
 use crate::parse::PResult;
 use crate::ThinVec;
 use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint};
+use crate::tokenstream::IsJoint::{self, NonJoint};
 use crate::symbol::{kw, sym, Symbol};
 use crate::parse::diagnostics::{Error, dummy_arg};
+use crate::util::parser::AssocOp;
 
 use errors::{Applicability, DiagnosticId, FatalError};
 use rustc_target::spec::abi::{self, Abi};
@@ -116,6 +118,8 @@ pub struct Parser<'a> {
     /// with non-interpolated identifier and lifetime tokens they refer to.
     /// Perhaps the normalized / non-normalized setup can be simplified somehow.
     pub token: Token,
+    /// Is `token` joined to the next one?
+    is_joint: IsJoint,
     /// Span of the current non-normalized token.
     meta_var_span: Option<Span>,
     /// Span of the previous non-normalized token.
@@ -213,21 +217,21 @@ impl TokenCursorFrame {
 }
 
 impl TokenCursor {
-    fn next(&mut self) -> Token {
+    fn next(&mut self) -> (Token, IsJoint) {
         loop {
-            let tree = if !self.frame.open_delim {
+            let (tree, is_joint) = if !self.frame.open_delim {
                 self.frame.open_delim = true;
-                TokenTree::open_tt(self.frame.span.open, self.frame.delim)
-            } else if let Some(tree) = self.frame.tree_cursor.next() {
-                tree
+                (TokenTree::open_tt(self.frame.span.open, self.frame.delim), NonJoint)
+            } else if let Some((tree, is_joint)) = self.frame.tree_cursor.next_with_joint() {
+                (tree, is_joint)
             } else if !self.frame.close_delim {
                 self.frame.close_delim = true;
-                TokenTree::close_tt(self.frame.span.close, self.frame.delim)
+                (TokenTree::close_tt(self.frame.span.close, self.frame.delim), NonJoint)
             } else if let Some(frame) = self.stack.pop() {
                 self.frame = frame;
                 continue
             } else {
-                return Token::new(token::Eof, DUMMY_SP);
+                return (Token::new(token::Eof, DUMMY_SP), NonJoint);
             };
 
             match self.frame.last_token {
@@ -236,7 +240,7 @@ impl TokenCursor {
             }
 
             match tree {
-                TokenTree::Token(token) => return token,
+                TokenTree::Token(token) => return (token, is_joint),
                 TokenTree::Delimited(sp, delim, tts) => {
                     let frame = TokenCursorFrame::new(sp, delim, &tts);
                     self.stack.push(mem::replace(&mut self.frame, frame));
@@ -245,10 +249,11 @@ impl TokenCursor {
         }
     }
 
-    fn next_desugared(&mut self) -> Token {
-        let (name, sp) = match self.next() {
+    fn next_desugared(&mut self) -> (Token, IsJoint) {
+        let (tok, is_joint) = self.next();
+        let (name, sp) = match tok {
             Token { kind: token::DocComment(name), span } => (name, span),
-            tok => return tok,
+            tok => return (tok, is_joint),
         };
 
         let stripped = strip_doc_comment_decoration(&name.as_str());
@@ -341,6 +346,7 @@ impl<'a> Parser<'a> {
         let mut parser = Parser {
             sess,
             token: Token::dummy(),
+            is_joint: IsJoint::NonJoint,
             prev_span: DUMMY_SP,
             meta_var_span: None,
             prev_token_kind: PrevTokenKind::Other,
@@ -370,7 +376,9 @@ impl<'a> Parser<'a> {
             subparser_name,
         };
 
-        parser.token = parser.next_tok();
+        let (token, is_joint) = parser.next_tok();
+        parser.token = token;
+        parser.is_joint = is_joint;
 
         if let Some(directory) = directory {
             parser.directory = directory;
@@ -387,8 +395,8 @@ impl<'a> Parser<'a> {
         parser
     }
 
-    fn next_tok(&mut self) -> Token {
-        let mut next = if self.desugar_doc_comments {
+    fn next_tok(&mut self) -> (Token, IsJoint) {
+        let (mut next, is_joint) = if self.desugar_doc_comments {
             self.token_cursor.next_desugared()
         } else {
             self.token_cursor.next()
@@ -397,7 +405,7 @@ impl<'a> Parser<'a> {
             // Tweak the location for better diagnostics, but keep syntactic context intact.
             next.span = self.prev_span.with_ctxt(next.span.ctxt());
         }
-        next
+        (next, is_joint)
     }
 
     /// Converts the current token to a string using `self`'s reader.
@@ -604,6 +612,54 @@ impl<'a> Parser<'a> {
         }
     }
 
+    fn assoc_op(&self) -> Option<AssocOp> {
+        use AssocOp::*;
+        let op = match self.token.kind {
+            token::BinOpEq(k) => AssignOp(k),
+            token::Eq => Assign,
+            token::BinOp(token::BinOpToken::Star) => Multiply,
+            token::BinOp(token::BinOpToken::Slash) => Divide,
+            token::BinOp(token::BinOpToken::Percent) => Modulus,
+            token::BinOp(token::BinOpToken::Plus) => Add,
+            token::BinOp(token::BinOpToken::Minus) => Subtract,
+            token::BinOp(token::BinOpToken::Shl) => ShiftLeft,
+            token::BinOp(token::BinOpToken::Shr) => ShiftRight,
+            token::BinOp(token::BinOpToken::And) => BitAnd,
+            token::BinOp(token::BinOpToken::Caret) => BitXor,
+            token::BinOp(token::BinOpToken::Or) => BitOr,
+            token::Lt => Less,
+            token::Le => LessEqual,
+            token::Ge => GreaterEqual,
+            token::Gt => Greater,
+            token::EqEq => Equal,
+            token::Not if self.look_ahead_joint(token::Eq) => NotEqual,
+            token::Ne => NotEqual,
+            token::AndAnd => LAnd,
+            token::OrOr => LOr,
+            token::DotDot => DotDot,
+            token::DotDotEq => DotDotEq,
+            // DotDotDot is no longer supported, but we need some way to display the error
+            token::DotDotDot => DotDotEq,
+            token::Colon => Colon,
+            // `<-` should probably be `< -`
+            token::LArrow => Less,
+            _ if self.token.is_keyword(kw::As) => As,
+            _ => return None
+        };
+        log::debug!("assoc_op() = {:?}", op);
+        Some(op)
+    }
+
+    fn bump_assoc_op(&mut self, op: AssocOp) {
+        match op {
+            AssocOp::NotEqual => {
+                self.bump();
+                self.bump();
+            }
+            _ => self.bump(),
+        }
+    }
+
     /// Checks to see if the next token is either `+` or `+=`.
     /// Otherwise returns `false`.
     fn check_plus(&mut self) -> bool {
@@ -891,8 +947,9 @@ impl<'a> Parser<'a> {
             token::Ident(..) => PrevTokenKind::Ident,
             _ => PrevTokenKind::Other,
         };
-
-        self.token = self.next_tok();
+        let (token, is_joint) = self.next_tok();
+        self.token = token;
+        self.is_joint = is_joint;
         self.expected_tokens.clear();
         // check after each token
         self.process_potential_macro_variable();
@@ -928,6 +985,16 @@ impl<'a> Parser<'a> {
         })
     }
 
+    fn look_ahead_joint(&self, kind: TokenKind) -> bool {
+        if self.is_joint == NonJoint {
+            return false;
+        }
+        match self.token_cursor.frame.tree_cursor.look_ahead(0) {
+            Some(TokenTree::Token(token)) => token.kind == kind,
+            _ => false,
+        }
+    }
+
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
     fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
         self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw)))

src/libsyntax/parse/parser/expr.rs

@@ -152,7 +152,7 @@ impl<'a> Parser<'a> {
         };
         let last_type_ascription_set = self.last_type_ascription.is_some();
 
-        match (self.expr_is_complete(&lhs), AssocOp::from_token(&self.token)) {
+        match (self.expr_is_complete(&lhs), self.assoc_op()) {
             (true, None) => {
                 self.last_type_ascription = None;
                 // Semi-statement forms are odd. See https://github.com/rust-lang/rust/issues/29071
@@ -197,7 +197,7 @@ impl<'a> Parser<'a> {
             }
         }
         self.expected_tokens.push(TokenType::Operator);
-        while let Some(op) = AssocOp::from_token(&self.token) {
+        while let Some(op) = self.assoc_op() {
 
             // Adjust the span for interpolated LHS to point to the `$lhs` token and not to what
             // it refers to. Interpolated identifiers are unwrapped early and never show up here
@@ -229,7 +229,7 @@ impl<'a> Parser<'a> {
                 self.err_larrow_operator(self.token.span);
             }
 
-            self.bump();
+            self.bump_assoc_op(op);
             if op.is_comparison() {
                 self.check_no_chained_comparison(&lhs, &op);
             }
@@ -363,14 +363,14 @@ impl<'a> Parser<'a> {
         debug_assert!([token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token.kind),
                       "parse_prefix_range_expr: token {:?} is not DotDot/DotDotEq",
                       self.token);
-        let tok = self.token.clone();
+        let assoc_op = self.assoc_op();
         let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
         let lo = self.token.span;
         let mut hi = self.token.span;
         self.bump();
         let opt_end = if self.is_at_start_of_range_notation_rhs() {
             // RHS must be parsed with more associativity than the dots.
-            let next_prec = AssocOp::from_token(&tok).unwrap().precedence() + 1;
+            let next_prec = assoc_op.unwrap().precedence() + 1;
             Some(self.parse_assoc_expr_with(next_prec, LhsExpr::NotYetParsed)
                 .map(|x| {
                     hi = x.span;
@@ -379,7 +379,7 @@ impl<'a> Parser<'a> {
         } else {
             None
         };
-        let limits = if tok == token::DotDot {
+        let limits = if assoc_op == Some(AssocOp::DotDot) {
             RangeLimits::HalfOpen
         } else {
             RangeLimits::Closed
@@ -1011,7 +1011,8 @@ impl<'a> Parser<'a> {
                     let path = self.parse_path(PathStyle::Expr)?;
 
                     // `!`, as an operator, is prefix, so we know this isn't that
-                    if self.eat(&token::Not) {
+                    if self.check(&token::Not) && self.look_ahead(1, |t| t.is_open_delim()) {
+                        self.bump();
                         // MACRO INVOCATION expression
                         let (delim, tts) = self.expect_delimited_token_tree()?;
                         hi = self.prev_span;

src/libsyntax/parse/token.rs

@@ -316,6 +316,13 @@ impl Token {
         }
     }
 
+    crate fn is_open_delim(&self) -> bool {
+        match self.kind {
+            OpenDelim(..) => true,
+            _ => false,
+        }
+    }
+
     crate fn is_like_plus(&self) -> bool {
         match self.kind {
             BinOp(Plus) | BinOpEq(Plus) => true,
@@ -548,7 +555,8 @@ impl Token {
         }
     }
 
-    crate fn glue(&self, joint: &Token) -> Option<Token> {
+    #[allow(unused)]
+    crate fn glue_for_tt_matcher(&self, joint: &Token) -> Option<Token> {
         let kind = match self.kind {
             Eq => match joint.kind {
                 Eq => EqEq,
@@ -608,7 +616,7 @@ impl Token {
         Some(Token::new(kind, self.span.to(joint.span)))
     }
 
-    crate fn glue_for_parser(&self, joint: &Token) -> Option<Token> {
+    crate fn glue(&self, joint: &Token) -> Option<Token> {
         let kind = match self.kind {
             Eq => match joint.kind {
                 Eq => EqEq,
@@ -628,10 +636,10 @@ impl Token {
                 Ge => BinOpEq(Shr),
                 _ => return None,
             },
-            Not => match joint.kind {
-                Eq => Ne,
-                _ => return None,
-            },
+            // Not => match joint.kind {
+            //     Eq => Ne,
+            //     _ => return None,
+            // },
             BinOp(op) => match joint.kind {
                 Eq => BinOpEq(op),
                 BinOp(And) if op == And => AndAnd,
@@ -663,6 +671,8 @@ impl Token {
             Question | OpenDelim(..) | CloseDelim(..) |
             Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
             Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None,
+
+            Not => return None,
         };
 
         Some(Token::new(kind, self.span.to(joint.span)))

src/libsyntax/util/parser.rs

@@ -1,11 +1,10 @@
-use crate::parse::token::{self, Token, BinOpToken};
-use crate::symbol::kw;
+use crate::parse::token::BinOpToken;
 use crate::ast::{self, BinOpKind};
 
 /// Associative operator with precedence.
 ///
 /// This is the enum which specifies operator precedence and fixity to the parser.
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 pub enum AssocOp {
     /// `+`
     Add,
@@ -68,42 +67,6 @@ pub enum Fixity {
 }
 
 impl AssocOp {
-    /// Creates a new AssocOP from a token
-    crate fn from_token(t: &Token) -> Option<AssocOp> {
-        use AssocOp::*;
-        match t.kind {
-            token::BinOpEq(k) => Some(AssignOp(k)),
-            token::Eq => Some(Assign),
-            token::BinOp(BinOpToken::Star) => Some(Multiply),
-            token::BinOp(BinOpToken::Slash) => Some(Divide),
-            token::BinOp(BinOpToken::Percent) => Some(Modulus),
-            token::BinOp(BinOpToken::Plus) => Some(Add),
-            token::BinOp(BinOpToken::Minus) => Some(Subtract),
-            token::BinOp(BinOpToken::Shl) => Some(ShiftLeft),
-            token::BinOp(BinOpToken::Shr) => Some(ShiftRight),
-            token::BinOp(BinOpToken::And) => Some(BitAnd),
-            token::BinOp(BinOpToken::Caret) => Some(BitXor),
-            token::BinOp(BinOpToken::Or) => Some(BitOr),
-            token::Lt => Some(Less),
-            token::Le => Some(LessEqual),
-            token::Ge => Some(GreaterEqual),
-            token::Gt => Some(Greater),
-            token::EqEq => Some(Equal),
-            token::Ne => Some(NotEqual),
-            token::AndAnd => Some(LAnd),
-            token::OrOr => Some(LOr),
-            token::DotDot => Some(DotDot),
-            token::DotDotEq => Some(DotDotEq),
-            // DotDotDot is no longer supported, but we need some way to display the error
-            token::DotDotDot => Some(DotDotEq),
-            token::Colon => Some(Colon),
-            // `<-` should probably be `< -`
-            token::LArrow => Some(Less),
-            _ if t.is_keyword(kw::As) => Some(As),
-            _ => None
-        }
-    }
-
     /// Creates a new AssocOp from ast::BinOpKind.
     pub fn from_ast_binop(op: BinOpKind) -> Self {
         use AssocOp::*;

src/libsyntax_ext/assert.rs

@@ -61,6 +61,7 @@ fn parse_assert<'a>(
     sp: Span,
     tts: &[TokenTree]
 ) -> Result<Assert, DiagnosticBuilder<'a>> {
+    log::debug!("parse_assert({:?})", tts);
     let mut parser = cx.new_parser_from_tts(tts);
 
     if parser.token == token::Eof {