Bring pycodestyle rules into full compatibility (on SciPy) (#4472)

crates/ruff/resources/test/fixtures/pycodestyle/E20.py

@@ -76,3 +76,11 @@
 a[b1, :] == a[b1, ...]
 b = a[:, b1]
 #:
+
+#: E201:1:6
+spam[ ~ham]
+
+#: Okay
+x = [  #
+    'some value',
+]

crates/ruff/resources/test/fixtures/pycodestyle/E22.py

@@ -169,4 +169,14 @@ def squares(n):
     -6: "\u03bc",  # Greek letter mu
     -3: "m",
 }
+
+i = (
+    i +  #
+    1
+)
+
+x[~y]
+
+if i == -1:
+    pass
 #:

crates/ruff/resources/test/fixtures/pycodestyle/E23.py

@@ -18,3 +18,13 @@ def foo() -> None:
     #: E231
     if (1,2):
         pass
+
+#: Okay
+a = (1,\
+2)
+
+#: E231:2:20
+mdtypes_template = {
+    'tag_full': [('mdtype', 'u4'), ('byte_count', 'u4')],
+    'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
+}

crates/ruff/resources/test/fixtures/pycodestyle/E27.py

@@ -56,3 +56,7 @@
 def f():
     print((yield))
     x = (yield)
+#: Okay
+if (a and
+    b):
+    pass

crates/ruff/src/checkers/logical_lines.rs

@@ -57,10 +57,11 @@ pub(crate) fn check_logical_lines(
 
         if line
             .flags()
-            .contains(TokenFlags::OPERATOR | TokenFlags::PUNCTUATION)
+            .intersects(TokenFlags::OPERATOR | TokenFlags::BRACKET | TokenFlags::PUNCTUATION)
         {
             extraneous_whitespace(&line, &mut context);
         }
+
         if line.flags().contains(TokenFlags::KEYWORD) {
             whitespace_around_keywords(&line, &mut context);
             missing_whitespace_after_keyword(&line, &mut context);

crates/ruff/src/rules/pycodestyle/rules/logical_lines/indentation.rs

@@ -219,12 +219,19 @@ impl Violation for UnexpectedIndentationComment {
 /// ## References
 /// - [PEP 8](https://peps.python.org/pep-0008/#indentation)
 #[violation]
-pub struct OverIndented;
+pub struct OverIndented {
+    is_comment: bool,
+}
 
 impl Violation for OverIndented {
     #[derive_message_formats]
     fn message(&self) -> String {
-        format!("Over-indented")
+        let OverIndented { is_comment } = self;
+        if *is_comment {
+            format!("Over-indented (comment)")
+        } else {
+            format!("Over-indented")
+        }
     }
 }
 
@@ -269,7 +276,12 @@ pub(crate) fn indentation(
         let expected_indent_amount = if indent_char == '\t' { 8 } else { 4 };
         let expected_indent_level = prev_indent_level.unwrap_or(0) + expected_indent_amount;
         if indent_level > expected_indent_level {
-            diagnostics.push(OverIndented.into());
+            diagnostics.push(
+                OverIndented {
+                    is_comment: logical_line.is_comment_only(),
+                }
+                .into(),
+            );
         }
     }
 

crates/ruff/src/rules/pycodestyle/rules/logical_lines/missing_whitespace.rs

@@ -54,7 +54,7 @@ pub(crate) fn missing_whitespace(
                 prev_lsqb = token.start();
             }
             TokenKind::Rsqb => {
-                open_parentheses += 1;
+                open_parentheses -= 1;
             }
             TokenKind::Lbrace => {
                 prev_lbrace = token.start();
@@ -63,7 +63,11 @@ pub(crate) fn missing_whitespace(
             TokenKind::Comma | TokenKind::Semi | TokenKind::Colon => {
                 let after = line.text_after(token);
 
-                if !after.chars().next().map_or(false, char::is_whitespace) {
+                if !after
+                    .chars()
+                    .next()
+                    .map_or(false, |c| char::is_whitespace(c) || c == '\\')
+                {
                     if let Some(next_token) = iter.peek() {
                         match (kind, next_token.kind()) {
                             (TokenKind::Colon, _)

crates/ruff/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs

@@ -31,7 +31,10 @@ pub(crate) fn missing_whitespace_after_keyword(
                 || matches!(tok0_kind, TokenKind::Async | TokenKind::Await)
                 || tok0_kind == TokenKind::Except && tok1_kind == TokenKind::Star
                 || tok0_kind == TokenKind::Yield && tok1_kind == TokenKind::Rpar
-                || matches!(tok1_kind, TokenKind::Colon | TokenKind::Newline))
+                || matches!(
+                    tok1_kind,
+                    TokenKind::Colon | TokenKind::Newline | TokenKind::NonLogicalNewline
+                ))
             && tok0.end() == tok1.start()
         {
             context.push(MissingWhitespaceAfterKeyword, tok0.range());

crates/ruff/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs

@@ -225,5 +225,7 @@ fn is_whitespace_needed(kind: TokenKind) -> bool {
             | TokenKind::Slash
             | TokenKind::Percent
     ) || kind.is_arithmetic()
-        || kind.is_bitwise_or_shift()
+        || (kind.is_bitwise_or_shift() &&
+            // As a special-case, pycodestyle seems to ignore whitespace around the tilde.
+            !matches!(kind, TokenKind::Tilde))
 }

crates/ruff/src/rules/pycodestyle/rules/logical_lines/mod.rs

@@ -355,7 +355,7 @@ impl LogicalLineToken {
     }
 }
 
-#[derive(Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub(crate) enum Whitespace {
     None,
     Single,
@@ -370,7 +370,10 @@ impl Whitespace {
         let mut has_tabs = false;
 
         for c in content.chars() {
-            if c == '\t' {
+            if c == '#' {
+                // Ignore leading whitespace between a token and an end-of-line comment
+                return (Whitespace::None, TextSize::default());
+            } else if c == '\t' {
                 has_tabs = true;
                 len += c.text_len();
             } else if matches!(c, '\n' | '\r') {

crates/ruff/src/rules/pycodestyle/snapshots/ruff__rules__pycodestyle__tests__E201_E20.py.snap

@@ -60,4 +60,13 @@ E20.py:12:15: E201 Whitespace after '{'
 16 | spam(ham[1], {eggs: 2})
    |
 
+E20.py:81:6: E201 Whitespace after '['
+   |
+81 | #: E201:1:6
+82 | spam[ ~ham]
+   |      ^ E201
+83 | 
+84 | #: Okay
+   |
+
 

crates/ruff/src/rules/pycodestyle/snapshots/ruff__rules__pycodestyle__tests__E231_E23.py.snap

@@ -78,5 +78,25 @@ E23.py:19:10: E231 [*] Missing whitespace after ','
 19    |-    if (1,2):
    19 |+    if (1, 2):
 20 20 |         pass
+21 21 | 
+22 22 | #: Okay
+
+E23.py:29:20: E231 [*] Missing whitespace after ':'
+   |
+29 | mdtypes_template = {
+30 |     'tag_full': [('mdtype', 'u4'), ('byte_count', 'u4')],
+31 |     'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
+   |                    ^ E231
+32 | }
+   |
+   = help: Added missing whitespace after ':'
+
+ℹ Suggested fix
+26 26 | #: E231:2:20
+27 27 | mdtypes_template = {
+28 28 |     'tag_full': [('mdtype', 'u4'), ('byte_count', 'u4')],
+29    |-    'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
+   29 |+    'tag_smalldata': [('byte_count_mdtype', 'u4'), ('data', 'S4')],
+30 30 | }
 
 

crates/ruff_python_ast/src/token_kind.rs

@@ -241,6 +241,7 @@ impl TokenKind {
                 | TokenKind::Percent
                 | TokenKind::Lbrace
                 | TokenKind::Rbrace
+                | TokenKind::EqEqual
                 | TokenKind::NotEqual
                 | TokenKind::LessEqual
                 | TokenKind::GreaterEqual