Use strip_suffix instead of slicing bytes

String::len won't necessarily produce correct results on arbitrary UTF8 strings
apache · Jan 12, 2025 · dfcd131 · dfcd131
1 parent e9cb408
commit dfcd131
Showing 1 changed file with 17 additions and 2 deletions.
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -1563,13 +1563,19 @@ impl<'a> Tokenizer<'a> {
                             temp.push(ch);
 
                             if temp.ends_with(&end_delimiter) {
-                                s.push_str(&temp[..temp.len() - end_delimiter.len()]);
+                                if let Some(temp) = temp.strip_suffix(&end_delimiter) {
+                                    s.push_str(temp);
+                                    break;
+                                }
                                 break;
                             }
                         }
                         None => {
                             if temp.ends_with(&end_delimiter) {
-                                s.push_str(&temp[..temp.len() - end_delimiter.len()]);
+                                if let Some(temp) = temp.strip_suffix(&end_delimiter) {
+                                    s.push_str(temp);
+                                    break;
+                                }
                                 break;
                             }
 
@@ -2579,6 +2585,15 @@ mod tests {
                     Token::Number("1".into(), false),
                 ]
             ),
+            (
+                String::from("$function$abc$q$data$q$$function$"),
+                vec![
+                    Token::DollarQuotedString(DollarQuotedString {
+                        value: "abc$q$data$q$".into(),
+                        tag: Some("function".into()),
+                    }),
+                ]
+            ),
         ];
 
         let dialect = GenericDialect {};