From dfcd1313e7a4a8ef7b97175832bdf9f10a3a531a Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Sun, 12 Jan 2025 18:31:25 +0100 Subject: [PATCH] Use strip_suffix instead of slicing bytes String::len won't necessarily produce correct results on arbitrary UTF8 strings --- src/tokenizer.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index e7b8802d1..dbd7d56b9 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1563,13 +1563,19 @@ impl<'a> Tokenizer<'a> { temp.push(ch); if temp.ends_with(&end_delimiter) { - s.push_str(&temp[..temp.len() - end_delimiter.len()]); + if let Some(temp) = temp.strip_suffix(&end_delimiter) { + s.push_str(temp); + break; + } break; } } None => { if temp.ends_with(&end_delimiter) { - s.push_str(&temp[..temp.len() - end_delimiter.len()]); + if let Some(temp) = temp.strip_suffix(&end_delimiter) { + s.push_str(temp); + break; + } break; } @@ -2579,6 +2585,15 @@ mod tests { Token::Number("1".into(), false), ] ), + ( + String::from("$function$abc$q$data$q$$function$"), + vec![ + Token::DollarQuotedString(DollarQuotedString { + value: "abc$q$data$q$".into(), + tag: Some("function".into()), + }), + ] + ), ]; let dialect = GenericDialect {};