chore: lex doc comments

compiler/noirc_frontend/src/lexer/lexer.rs

@@ -1,4 +1,4 @@
-use crate::token::Attribute;
+use crate::token::{Attribute, DocStyle};
 
 use super::{
     errors::LexerErrorKind,
@@ -67,6 +67,13 @@ impl<'a> Lexer<'a> {
         self.char_iter.peek().map(|(c, _)| *c)
     }
 
+    /// Peeks at the character two positions ahead. Does not iterate the cursor
+    fn peek2_char(&mut self) -> Option<char> {
+        let mut chars = self.char_iter.clone();
+        chars.next();
+        chars.next().map(|(c, _)| c)
+    }
+
     /// Peeks at the next char and returns true if it is equal to the char argument
     fn peek_char_is(&mut self, ch: char) -> bool {
         self.peek_char() == Some(ch)
@@ -388,14 +395,39 @@ impl<'a> Lexer<'a> {
     }
 
     fn parse_comment(&mut self, start: u32) -> SpannedTokenResult {
+        let doc_style = match self.peek_char() {
+            Some('!') => {
+                self.next_char();
+                Some(DocStyle::Inner)
+            }
+            Some('/') if self.peek2_char() != '/'.into() => {
+                self.next_char();
+                Some(DocStyle::Outer)
+            }
+            _ => None,
+        };
         let comment = self.eat_while(None, |ch| ch != '\n');
+
         if self.skip_comments {
             return self.next_token();
         }
-        Ok(Token::LineComment(comment).into_span(start, self.position))
+
+        Ok(Token::LineComment(comment, doc_style).into_span(start, self.position))
     }
 
     fn parse_block_comment(&mut self, start: u32) -> SpannedTokenResult {
+        let doc_style = match self.peek_char() {
+            Some('!') => {
+                self.next_char();
+                Some(DocStyle::Inner)
+            }
+            Some('*') if !matches!(self.peek2_char(), Some('*' | '/')) => {
+                self.next_char();
+                Some(DocStyle::Outer)
+            }
+            _ => None,
+        };
+
         let mut depth = 1usize;
 
         let mut content = String::new();
@@ -424,7 +456,7 @@ impl<'a> Lexer<'a> {
             if self.skip_comments {
                 return self.next_token();
             }
-            Ok(Token::BlockComment(content).into_span(start, self.position))
+            Ok(Token::BlockComment(content, doc_style).into_span(start, self.position))
         } else {
             let span = Span::inclusive(start, self.position);
             Err(LexerErrorKind::UnterminatedBlockComment { span })
@@ -734,6 +766,32 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_comments() {
+        let input = "
+            // comment
+            /// comment
+            //! comment
+            /* comment */
+            /** outer doc block */
+            /*! inner doc block */
+        ";
+        let expected = [
+            Token::LineComment(" comment".into(), None),
+            Token::LineComment(" comment".into(), DocStyle::Outer.into()),
+            Token::LineComment(" comment".into(), DocStyle::Inner.into()),
+            Token::BlockComment(" comment ".into(), None),
+            Token::BlockComment(" outer doc block ".into(), DocStyle::Outer.into()),
+            Token::BlockComment(" inner doc block ".into(), DocStyle::Inner.into()),
+        ];
+
+        let mut lexer = Lexer::new(input).skip_comments(false);
+        for token in expected {
+            let first_lexer_output = lexer.next_token().unwrap();
+            assert_eq!(token, first_lexer_output);
+        }
+    }
+
     #[test]
     fn test_nested_block_comments() {
         let input = "

compiler/noirc_frontend/src/lexer/token.rs

@@ -19,8 +19,8 @@ pub enum Token {
     Keyword(Keyword),
     IntType(IntType),
     Attribute(Attribute),
-    LineComment(String),
-    BlockComment(String),
+    LineComment(String, Option<DocStyle>),
+    BlockComment(String, Option<DocStyle>),
     /// <
     Less,
     /// <=
@@ -97,6 +97,12 @@ pub enum Token {
     Invalid(char),
 }
 
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
+pub enum DocStyle {
+    Outer,
+    Inner,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct SpannedToken(Spanned<Token>);
 
@@ -151,8 +157,8 @@ impl fmt::Display for Token {
             Token::FmtStr(ref b) => write!(f, "f{b}"),
             Token::Keyword(k) => write!(f, "{k}"),
             Token::Attribute(ref a) => write!(f, "{a}"),
-            Token::LineComment(ref s) => write!(f, "//{s}"),
-            Token::BlockComment(ref s) => write!(f, "/*{s}*/"),
+            Token::LineComment(ref s, _style) => write!(f, "//{s}"),
+            Token::BlockComment(ref s, _style) => write!(f, "/*{s}*/"),
             Token::IntType(ref i) => write!(f, "{i}"),
             Token::Less => write!(f, "<"),
             Token::LessEqual => write!(f, "<="),

tooling/nargo_fmt/src/utils.rs

@@ -18,7 +18,9 @@
 
 pub(crate) fn comments(source: &str) -> impl Iterator<Item = String> + '_ {
     Lexer::new(source).skip_comments(false).flatten().filter_map(|spanned| {
-        if let Token::LineComment(content) | Token::BlockComment(content) = spanned.into_token() {
+        if let Token::LineComment(content, _) | Token::BlockComment(content, _) =
+            spanned.into_token()
+        {
             Some(content)
         } else {
             None
@@ -89,7 +91,7 @@
         let starts_with_single_line_comment = leading_trimmed.starts_with("//");
 
         if ends_with_block_comment {
             let comment_end = leading_trimmed.rfind(|c| c == '/').unwrap();
 
             if leading[comment_end..].contains('\n') {
                 newlines = true;
@@ -135,7 +137,7 @@
     fn find_comment_end(slice: &str) -> usize {
         slice
             .find_token_with(|token| {
-                matches!(token, Token::LineComment(_) | Token::BlockComment(_))
+                matches!(token, Token::LineComment(_, _) | Token::BlockComment(_, _))
             })
             .map(|index| index as usize)
             .unwrap_or(slice.len())