Add support for PEP 701 (#7376)

## Summary

This PR adds support for PEP 701 in Ruff. This is a rollup PR of all the
other individual PRs. The separate PRs were created for logic separation
and code reviews. Refer to each pull request for a detail description on
the change.

Refer to the PR description for the list of pull requests within this PR.

## Test Plan

### Formatter ecosystem checks

Explanation for the change in ecosystem check:
#7597 (comment)

#### `main`

```
| project      | similarity index  | total files       | changed files     |
|--------------|------------------:|------------------:|------------------:|
| cpython      |           0.76083 |              1789 |              1631 |
| django       |           0.99983 |              2760 |                36 |
| transformers |           0.99963 |              2587 |               319 |
| twine        |           1.00000 |                33 |                 0 |
| typeshed     |           0.99983 |              3496 |                18 |
| warehouse    |           0.99967 |               648 |                15 |
| zulip        |           0.99972 |              1437 |                21 |
```

#### `dhruv/pep-701`

```
| project      | similarity index  | total files       | changed files     |
|--------------|------------------:|------------------:|------------------:|
| cpython      |           0.76051 |              1789 |              1632 |
| django       |           0.99983 |              2760 |                36 |
| transformers |           0.99963 |              2587 |               319 |
| twine        |           1.00000 |                33 |                 0 |
| typeshed     |           0.99983 |              3496 |                18 |
| warehouse    |           0.99967 |               648 |                15 |
| zulip        |           0.99972 |              1437 |                21 |
```

crates/ruff_benchmark/benches/formatter.rs

@@ -65,7 +65,7 @@ fn benchmark_formatter(criterion: &mut Criterion) {
                 let comment_ranges = comment_ranges.finish();
 
                 // Parse the AST.
-                let module = parse_tokens(tokens, Mode::Module, "<filename>")
+                let module = parse_tokens(tokens, case.code(), Mode::Module, "<filename>")
                     .expect("Input to be a valid python program");
 
                 b.iter(|| {

crates/ruff_linter/resources/test/fixtures/flake8_implicit_str_concat/ISC.py

@@ -59,3 +59,23 @@
 _ = foo + bar + "abc"
 _ = "abc" + foo + bar
 _ = foo + "abc" + bar
+
+# Multiple strings nested inside a f-string
+_ = f"a {'b' 'c' 'd'} e"
+_ = f"""abc {"def" "ghi"} jkl"""
+_ = f"""abc {
+    "def"
+    "ghi"
+} jkl"""
+
+# Nested f-strings
+_ = "a" f"b {f"c" f"d"} e" "f"
+_ = f"b {f"c" f"d {f"e" f"f"} g"} h"
+_ = f"b {f"abc" \
+    f"def"} g"
+
+# Explicitly concatenated nested f-strings
+_ = f"a {f"first"
+    + f"second"} d"
+_ = f"a {f"first {f"middle"}"
+    + f"second"} d"

crates/ruff_linter/resources/test/fixtures/flake8_quotes/doubles_escaped.py

@@ -9,3 +9,33 @@
     'This is a'
     '\'string\''
 )
+
+# Same as above, but with f-strings
+f'This is a \'string\''  # Q003
+f'This is \\ a \\\'string\''  # Q003
+f'"This" is a \'string\''
+f"This is a 'string'"
+f"\"This\" is a 'string'"
+fr'This is a \'string\''
+fR'This is a \'string\''
+foo = (
+    f'This is a'
+    f'\'string\''  # Q003
+)
+
+# Nested f-strings (Python 3.12+)
+#
+# The first one is interesting because the fix for it is valid pre 3.12:
+#
+#   f"'foo' {'nested'}"
+#
+# but as the actual string itself is invalid pre 3.12, we don't catch it.
+f'\'foo\' {'nested'}'  # Q003
+f'\'foo\' {f'nested'}'  # Q003
+f'\'foo\' {f'\'nested\''} \'\''  # Q003
+
+f'normal {f'nested'} normal'
+f'\'normal\' {f'nested'} normal'  # Q003
+f'\'normal\' {f'nested'} "double quotes"'
+f'\'normal\' {f'\'nested\' {'other'} normal'} "double quotes"'  # Q003
+f'\'normal\' {f'\'nested\' {'other'} "double quotes"'} normal'  # Q00l

crates/ruff_linter/resources/test/fixtures/flake8_quotes/singles_escaped.py

@@ -8,3 +8,32 @@
     "This is a"
     "\"string\""
 )
+
+# Same as above, but with f-strings
+f"This is a \"string\""
+f"'This' is a \"string\""
+f'This is a "string"'
+f'\'This\' is a "string"'
+fr"This is a \"string\""
+fR"This is a \"string\""
+foo = (
+    f"This is a"
+    f"\"string\""
+)
+
+# Nested f-strings (Python 3.12+)
+#
+# The first one is interesting because the fix for it is valid pre 3.12:
+#
+#   f'"foo" {"nested"}'
+#
+# but as the actual string itself is invalid pre 3.12, we don't catch it.
+f"\"foo\" {"foo"}"  # Q003
+f"\"foo\" {f"foo"}"  # Q003
+f"\"foo\" {f"\"foo\""} \"\""  # Q003
+
+f"normal {f"nested"} normal"
+f"\"normal\" {f"nested"} normal"  # Q003
+f"\"normal\" {f"nested"} 'single quotes'"
+f"\"normal\" {f"\"nested\" {"other"} normal"} 'single quotes'"  # Q003
+f"\"normal\" {f"\"nested\" {"other"} 'single quotes'"} normal"  # Q003

crates/ruff_linter/resources/test/fixtures/pycodestyle/E20.py

@@ -84,3 +84,8 @@
 x = [  #
     'some value',
 ]
+
+# F-strings
+f"{ {'a': 1} }"
+f"{[ { {'a': 1} } ]}"
+f"normal { {f"{ { [1, 2] } }" } } normal"

crates/ruff_linter/resources/test/fixtures/pycodestyle/E23.py

@@ -29,5 +29,16 @@ def foo() -> None:
     'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
 }
 
+# E231
+f"{(a,b)}"
+
+# Okay because it's hard to differentiate between the usages of a colon in a f-string
+f"{a:=1}"
+f"{ {'a':1} }"
+f"{a:.3f}"
+f"{(a:=1)}"
+f"{(lambda x:x)}"
+f"normal{f"{a:.3f}"}normal"
+
 #: Okay
 a = (1,

crates/ruff_linter/resources/test/fixtures/pycodestyle/E25.py

@@ -48,3 +48,9 @@ def add(a: int=0, b: int =0, c: int= 0) -> int:
 #: Okay
 def add(a: int = _default(name='f')):
     return a
+
+# F-strings
+f"{a=}"
+f"{a:=1}"
+f"{foo(a=1)}"
+f"normal {f"{a=}"} normal"

crates/ruff_linter/resources/test/fixtures/pycodestyle/W19.py

@@ -152,3 +152,11 @@ def test_keys(self):
 	multiline string with tab in it, different lines
 '''
 "	single line string with tab in it"
+
+f"test{
+	tab_indented_should_be_flagged
+}	<- this tab is fine"
+
+f"""test{
+	tab_indented_should_be_flagged
+}	<- this tab is fine"""

crates/ruff_linter/resources/test/fixtures/pycodestyle/W605_2.py

@@ -0,0 +1,54 @@
+# Same as `W605_0.py` but using f-strings instead.
+
+#: W605:1:10
+regex = f'\.png$'
+
+#: W605:2:1
+regex = f'''
+\.png$
+'''
+
+#: W605:2:6
+f(
+    f'\_'
+)
+
+#: W605:4:6
+f"""
+multi-line
+literal
+with \_ somewhere
+in the middle
+"""
+
+#: W605:1:38
+value = f'new line\nand invalid escape \_ here'
+
+
+#: Okay
+regex = fr'\.png$'
+regex = f'\\.png$'
+regex = fr'''
+\.png$
+'''
+regex = fr'''
+\\.png$
+'''
+s = f'\\'
+regex = f'\w'  # noqa
+regex = f'''
+\w
+'''  # noqa
+
+regex = f'\\\_'
+value = f'\{{1}}'
+value = f'\{1}'
+value = f'{1:\}'
+value = f"{f"\{1}"}"
+value = rf"{f"\{1}"}"
+
+# Okay
+value = rf'\{{1}}'
+value = rf'\{1}'
+value = rf'{1:\}'
+value = f"{rf"\{1}"}"

crates/ruff_linter/resources/test/fixtures/pyflakes/F541.py

@@ -40,7 +40,5 @@
 ""f""
 ''f""
 (""f""r"")
-
-# To be fixed
-# Error: f-string: single '}' is not allowed at line 41 column 8
-# f"\{{x}}"
+f"{v:{f"0.2f"}}"
+f"\{{x}}"

crates/ruff_linter/resources/test/fixtures/ruff/confusables.py

@@ -29,3 +29,19 @@ def f():
 # consisting of a single ambiguous character, while the second character is a "word
 # boundary" (whitespace) that it itself ambiguous.
 x = "Р усский"
+
+# Same test cases as above but using f-strings instead:
+x = f"𝐁ad string"
+x = f"−"
+x = f"Русский"
+x = f"βα Bαd"
+x = f"Р усский"
+
+# Nested f-strings
+x = f"𝐁ad string {f" {f"Р усский"}"}"
+
+# Comments inside f-strings
+x = f"string { # And here's a comment with an unusual parenthesis: ）
+# And here's a comment with a greek alpha: ∗
+foo # And here's a comment with an unusual punctuation mark: ᜵
+}"

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -966,9 +966,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                 pylint::rules::await_outside_async(checker, expr);
             }
         }
-        Expr::FString(ast::ExprFString { values, .. }) => {
+        Expr::FString(fstring @ ast::ExprFString { values, .. }) => {
             if checker.enabled(Rule::FStringMissingPlaceholders) {
-                pyflakes::rules::f_string_missing_placeholders(expr, values, checker);
+                pyflakes::rules::f_string_missing_placeholders(fstring, checker);
             }
             if checker.enabled(Rule::HardcodedSQLExpression) {
                 flake8_bandit::rules::hardcoded_sql_expression(checker, expr);

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -183,7 +183,7 @@ impl<'a> Checker<'a> {
 
         // Find the quote character used to start the containing f-string.
         let expr = self.semantic.current_expression()?;
-        let string_range = self.indexer.f_string_range(expr.start())?;
+        let string_range = self.indexer.fstring_ranges().innermost(expr.start())?;
         let trailing_quote = trailing_quote(self.locator.slice(string_range))?;
 
         // Invert the quote character, if it's a single quote.

crates/ruff_linter/src/checkers/tokens.rs

@@ -45,23 +45,25 @@ pub(crate) fn check_tokens(
         let mut state_machine = StateMachine::default();
         for &(ref tok, range) in tokens.iter().flatten() {
             let is_docstring = state_machine.consume(tok);
-            if matches!(tok, Tok::String { .. } | Tok::Comment(_)) {
-                ruff::rules::ambiguous_unicode_character(
-                    &mut diagnostics,
-                    locator,
-                    range,
-                    if tok.is_string() {
-                        if is_docstring {
-                            Context::Docstring
-                        } else {
-                            Context::String
-                        }
+            let context = match tok {
+                Tok::String { .. } => {
+                    if is_docstring {
+                        Context::Docstring
                     } else {
-                        Context::Comment
-                    },
-                    settings,
-                );
-            }
+                        Context::String
+                    }
+                }
+                Tok::FStringMiddle { .. } => Context::String,
+                Tok::Comment(_) => Context::Comment,
+                _ => continue,
+            };
+            ruff::rules::ambiguous_unicode_character(
+                &mut diagnostics,
+                locator,
+                range,
+                context,
+                settings,
+            );
         }
     }
 
@@ -75,14 +77,14 @@ pub(crate) fn check_tokens(
 
     if settings.rules.enabled(Rule::InvalidEscapeSequence) {
         for (tok, range) in tokens.iter().flatten() {
-            if tok.is_string() {
-                pycodestyle::rules::invalid_escape_sequence(
-                    &mut diagnostics,
-                    locator,
-                    *range,
-                    settings.rules.should_fix(Rule::InvalidEscapeSequence),
-                );
-            }
+            pycodestyle::rules::invalid_escape_sequence(
+                &mut diagnostics,
+                locator,
+                indexer,
+                tok,
+                *range,
+                settings.rules.should_fix(Rule::InvalidEscapeSequence),
+            );
         }
     }
 
@@ -98,9 +100,7 @@ pub(crate) fn check_tokens(
         Rule::InvalidCharacterZeroWidthSpace,
     ]) {
         for (tok, range) in tokens.iter().flatten() {
-            if tok.is_string() {
-                pylint::rules::invalid_string_characters(&mut diagnostics, *range, locator);
-            }
+            pylint::rules::invalid_string_characters(&mut diagnostics, tok, *range, locator);
         }
     }
 
@@ -118,13 +118,16 @@ pub(crate) fn check_tokens(
         );
     }
 
+    if settings.rules.enabled(Rule::AvoidableEscapedQuote) && settings.flake8_quotes.avoid_escape {
+        flake8_quotes::rules::avoidable_escaped_quote(&mut diagnostics, tokens, locator, settings);
+    }
+
     if settings.rules.any_enabled(&[
         Rule::BadQuotesInlineString,
         Rule::BadQuotesMultilineString,
         Rule::BadQuotesDocstring,
-        Rule::AvoidableEscapedQuote,
     ]) {
-        flake8_quotes::rules::from_tokens(&mut diagnostics, tokens, locator, settings);
+        flake8_quotes::rules::check_string_quotes(&mut diagnostics, tokens, locator, settings);
     }
 
     if settings.rules.any_enabled(&[
@@ -136,6 +139,7 @@ pub(crate) fn check_tokens(
             tokens,
             &settings.flake8_implicit_str_concat,
             locator,
+            indexer,
         );
     }