Used pending tokens in replace with ##

Add active flag to HashHash so it does not get replaced indefinitely Ignore test which relies on jyn514#513 - stringify out of order
hdamron17 · Aug 17, 2020 · 6396e51 · 6396e51
1 parent 4f0e7d6
commit 6396e51
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 10 deletions.
diff --git a/src/data/lex.rs b/src/data/lex.rs
@@ -231,9 +231,9 @@ pub enum Token {
 
     // Misc
     Ellipsis,
-    StructDeref, // ->
-    Hash,        // #, used for preprocessing
-    HashHash,    // ##, used for preprocessing
+    StructDeref,    // ->
+    Hash,           // #, used for preprocessing
+    HashHash(bool), // ##, used for preprocessing (the bool is true unless it is created by `# ## #`)
 }
 
 /* impls */
@@ -394,7 +394,7 @@ impl std::fmt::Display for Token {
             Ellipsis => write!(f, "..."),
             StructDeref => write!(f, "->"),
             Hash => write!(f, "#"),
-            HashHash => write!(f, "##"),
+            HashHash(_) => write!(f, "##"),
         }
     }
 }

diff --git a/src/lex/cpp.rs b/src/lex/cpp.rs
@@ -1959,7 +1959,7 @@ h",
         assert_concat("+", "+", Some(PlusPlus));
         assert_concat(">>", "=", Some(Assignment(ShrEqual)));
         assert_concat(">", "=", Some(Comparison(GreaterEqual)));
-        assert_concat("#", "#", Some(HashHash));
+        assert_concat("#", "#", Some(HashHash(false)));
         assert_concat("-", ">", Some(StructDeref));
         assert_concat("const", "ance", Some(Id("constance".into())));
         assert_concat("xyz", "123", Some(Id("xyz123".into())));
@@ -1974,7 +1974,10 @@ h",
         assert_concat(r#""x""#, r#""y""#, None);
         assert_concat("0b1", "6", None);
         assert_concat("/", "/", None); // Not a comment
-
+    }
+    #[test]
+    #[ignore] // Related to https://github.com/jyn514/saltwater/issues/513
+    fn hash_and_hashhash() {
         assert_same(
             "#define     hash_hash # ## #
              #define     mkstr(a) # a

diff --git a/src/lex/mod.rs b/src/lex/mod.rs
@@ -365,7 +365,7 @@ impl Iterator for Lexer {
                 '#' => match self.peek() {
                     Some('#') => {
                         self.next_char();
-                        Token::HashHash
+                        Token::HashHash(true)
                     }
                     _ => Token::Hash,
                 },

diff --git a/src/lex/replace.rs b/src/lex/replace.rs
@@ -193,7 +193,7 @@ pub fn replace(
                             .into(),
                         )
                     });
-                replacements.push(concat_token); // TODO don't bypass pending
+                pending.push_back(concat_token);
                 continue;
             }
             Ok(Locatable {
@@ -245,7 +245,7 @@ pub fn replace(
                 }
             }
             Ok(Locatable {
-                data: Token::HashHash,
+                data: Token::HashHash(true),
                 ..
             }) => {
                 let preceding_tok = loop {
@@ -492,7 +492,13 @@ fn stringify(args: Vec<Token>) -> Token {
 fn concat(x: &Token, y: &Token, location: &Location) -> Option<Locatable<Token>> {
     let mut lexer = Lexer::new(location.file, format!("{}{}", x, y), false);
     match lexer.next() {
-        Some(Ok(tok)) if lexer.next().is_none() => Some(tok),
+        Some(Ok(tok)) if lexer.next().is_none() => Some(match tok {
+            Locatable {
+                data: Token::HashHash(_),
+                location,
+            } => location.with(Token::HashHash(false)),
+            tok => tok,
+        }),
         _ => None,
     }
 }