fix!: several format string fixes and improvements

compiler/noirc_frontend/src/lexer/lexer.rs

@@ -47,7 +47,7 @@
             done: false,
             skip_comments: true,
             skip_whitespaces: true,
             max_integer: BigInt::from_biguint(num_bigint::Sign::Plus, FieldElement::modulus())
                 - BigInt::one(),
         }
     }
@@ -409,6 +409,24 @@
 
     fn eat_string_literal(&mut self) -> SpannedTokenResult {
         let start = self.position;
+        let string = self.eat_string(start)?;
+        let str_literal_token = Token::Str(string);
+        let end = self.position;
+        Ok(str_literal_token.into_span(start, end))
+    }
+
+    // This differs from `eat_string_literal` in that we want the leading `f` to be captured in the Span
+    fn eat_fmt_string(&mut self) -> SpannedTokenResult {
+        let start = self.position;
+        self.next_char();
+
+        let string = self.eat_string(start)?;
+        let str_literal_token = Token::FmtStr(string);
+        let end = self.position;
+        Ok(str_literal_token.into_span(start, end))
+    }
+
+    fn eat_string(&mut self, start: u32) -> Result<String, LexerErrorKind> {
         let mut string = String::new();
 
         while let Some(next) = self.next_char() {
@@ -436,26 +454,7 @@
             string.push(char);
         }
 
-        let str_literal_token = Token::Str(string);
-
-        let end = self.position;
-        Ok(str_literal_token.into_span(start, end))
-    }
-
-    // This differs from `eat_string_literal` in that we want the leading `f` to be captured in the Span
-    fn eat_fmt_string(&mut self) -> SpannedTokenResult {
-        let start = self.position;
-
-        self.next_char();
-
-        let str_literal = self.eat_while(None, |ch| ch != '"');
-
-        let str_literal_token = Token::FmtStr(str_literal);
-
-        self.next_char(); // Advance past the closing quote
-
-        let end = self.position;
-        Ok(str_literal_token.into_span(start, end))
+        Ok(string)
     }
 
     fn eat_format_string_or_alpha_numeric(&mut self) -> SpannedTokenResult {
@@ -962,6 +961,60 @@
         }
     }
 
+    #[test]
+    fn test_eat_string_literal_with_escapes() {
+        let input = "let _word = \"hello\\n\\t\"";
+
+        let expected = vec![
+            Token::Keyword(Keyword::Let),
+            Token::Ident("_word".to_string()),
+            Token::Assign,
+            Token::Str("hello\n\t".to_string()),
+        ];
+        let mut lexer = Lexer::new(input);
+
+        for token in expected.into_iter() {
+            let got = lexer.next_token().unwrap();
+            assert_eq!(got, token);
+        }
+    }
+
+    #[test]
+    fn test_eat_fmt_string_literal() {
+        let input = "let _word = f\"hello\"";
+
+        let expected = vec![
+            Token::Keyword(Keyword::Let),
+            Token::Ident("_word".to_string()),
+            Token::Assign,
+            Token::FmtStr("hello".to_string()),
+        ];
+        let mut lexer = Lexer::new(input);
+
+        for token in expected.into_iter() {
+            let got = lexer.next_token().unwrap();
+            assert_eq!(got, token);
+        }
+    }
+
+    #[test]
+    fn test_eat_fmt_string_literal_with_escapes() {
+        let input = "let _word = f\"hello\\n\\t\"";
+
+        let expected = vec![
+            Token::Keyword(Keyword::Let),
+            Token::Ident("_word".to_string()),
+            Token::Assign,
+            Token::FmtStr("hello\n\t".to_string()),
+        ];
+        let mut lexer = Lexer::new(input);
+
+        for token in expected.into_iter() {
+            let got = lexer.next_token().unwrap();
+            assert_eq!(got, token);
+        }
+    }
+
     #[test]
     fn test_eat_integer_literals() {
         let test_cases: Vec<(&str, Token)> = vec![
@@ -1107,7 +1160,7 @@
     //   (expected_token_discriminator, strings_to_lex)
     // expected_token_discriminator matches a given token when
     // std::mem::discriminant returns the same discriminant for both.
     fn blns_base64_to_statements(base64_str: String) -> Vec<(Option<Token>, Vec<String>)> {
         use base64::engine::general_purpose;
         use std::borrow::Cow;
         use std::io::Cursor;
@@ -1165,13 +1218,13 @@
     fn test_big_list_of_naughty_strings() {
         use std::mem::discriminant;
 
         let blns_contents = include_str!("./blns/blns.base64.json");
         let blns_base64: Vec<String> =
             serde_json::from_str(blns_contents).expect("BLNS json invalid");
         for blns_base64_str in blns_base64 {
             let statements = blns_base64_to_statements(blns_base64_str);
             for (token_discriminator_opt, blns_program_strs) in statements {
                 for blns_program_str in blns_program_strs {
                     let mut expected_token_found = false;
                     let mut lexer = Lexer::new(&blns_program_str);
                     let mut result_tokens = Vec::new();
@@ -1257,7 +1310,7 @@
 
     #[test]
     fn test_non_ascii_comments() {
         let cases = vec!["// 🙂", "// schön", "/* in the middle 🙂 of a comment */"];
 
         for source in cases {
             let mut lexer = Lexer::new(source);