Skip to content

Commit

Permalink
Token Kind, remove text
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaReiser
MichaReiser committed Mar 26, 2023
1 parent 1eaefee commit 4224bfe
Show file tree
Hide file tree
Showing 17 changed files with 1,178 additions and 594 deletions.
47 changes: 25 additions & 22 deletions crates/ruff/src/checkers/logical_lines.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,11 @@ pub fn check_logical_lines(
) -> Vec<Diagnostic> {
let mut diagnostics = vec![];

let indent_char = stylist.indentation().as_char();
let mut prev_line = None;
let mut prev_indent_level = None;
for line in &LogicalLines::from_tokens(tokens, locator) {
// Extract the indentation level.
let Some(start_loc) = line.first_token_location() else { continue; };
let start_line = locator.slice(Range::new(Location::new(start_loc.row(), 0), *start_loc));
let indent_level = expand_indent(start_line);
let indent_size = 4;
let indent_char = stylist.indentation().as_char();

for line in &LogicalLines::from_tokens(tokens, locator) {
if line.flags().contains(TokenFlags::OPERATOR) {
for (location, kind) in space_around_operator(line.tokens(), locator) {
if settings.rules.enabled(kind.rule()) {
Expand Down Expand Up @@ -150,9 +145,7 @@ pub fn check_logical_lines(
#[cfg(not(feature = "logical_lines"))]
let should_fix = false;

for diagnostic in
missing_whitespace(line.text(), start_loc.row(), should_fix, indent_level)
{
for diagnostic in missing_whitespace(line.tokens(), locator, should_fix) {
if settings.rules.enabled(diagnostic.kind.rule()) {
diagnostics.push(diagnostic);
}
Expand All @@ -174,16 +167,20 @@ pub fn check_logical_lines(
}
}

for (index, kind) in indentation(
// Extract the indentation level.
let Some(start_loc) = line.first_token_location() else { continue; };
let start_line = locator.slice(Range::new(Location::new(start_loc.row(), 0), start_loc));
let indent_level = expand_indent(start_line);
let indent_size = 4;

for (location, kind) in indentation(
&line,
prev_line.as_ref(),
indent_char,
indent_level,
prev_indent_level,
indent_size,
) {
let (token_offset, pos) = line.mapping(index);
let location = Location::new(pos.row(), pos.column() + index - token_offset);
if settings.rules.enabled(kind.rule()) {
diagnostics.push(Diagnostic {
kind,
Expand All @@ -195,7 +192,7 @@ pub fn check_logical_lines(
}
}

if !line.is_comment() {
if !line.is_comment_only() {
prev_line = Some(line);
prev_indent_level = Some(indent_level);
}
Expand All @@ -221,7 +218,7 @@ z = x + 1"#;
let locator = Locator::new(contents);
let actual: Vec<String> = LogicalLines::from_tokens(&lxr, &locator)
.into_iter()
.map(|line| line.text().to_string())
.map(|line| line.text_trimmed().to_string())
.collect();
let expected = vec![
"x = 1".to_string(),
Expand All @@ -242,10 +239,10 @@ z = x + 1"#;
let locator = Locator::new(contents);
let actual: Vec<String> = LogicalLines::from_tokens(&lxr, &locator)
.into_iter()
.map(|line| line.text().to_string())
.map(|line| line.text_trimmed().to_string())
.collect();
let expected = vec![
"x = [1, 2, 3, ]".to_string(),
"x = [\n 1,\n 2,\n 3,\n]".to_string(),
"y = 2".to_string(),
"z = x + 1".to_string(),
];
Expand All @@ -256,9 +253,9 @@ z = x + 1"#;
let locator = Locator::new(contents);
let actual: Vec<String> = LogicalLines::from_tokens(&lxr, &locator)
.into_iter()
.map(|line| line.text().to_string())
.map(|line| line.text_trimmed().to_string())
.collect();
let expected = vec!["x = \"xxx\"".to_string()];
let expected = vec!["x = 'abc'".to_string()];
assert_eq!(actual, expected);

let contents = r#"
Expand All @@ -269,7 +266,7 @@ f()"#;
let locator = Locator::new(contents);
let actual: Vec<String> = LogicalLines::from_tokens(&lxr, &locator)
.into_iter()
.map(|line| line.text().to_string())
.map(|line| line.text_trimmed().to_string())
.collect();
let expected = vec!["def f():", "x = 1", "f()"];
assert_eq!(actual, expected);
Expand All @@ -284,9 +281,15 @@ f()"#;
let locator = Locator::new(contents);
let actual: Vec<String> = LogicalLines::from_tokens(&lxr, &locator)
.into_iter()
.map(|line| line.text().to_string())
.map(|line| line.text_trimmed().to_string())
.collect();
let expected = vec!["def f():", "\"xxxxxxxxxxxxxxxxxxxx\"", "", "x = 1", "f()"];
let expected = vec![
"def f():",
"\"\"\"Docstring goes here.\"\"\"",
"",
"x = 1",
"f()",
];
assert_eq!(actual, expected);
}
}
150 changes: 10 additions & 140 deletions crates/ruff/src/rules/pycodestyle/helpers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use rustpython_parser::Tok;

use ruff_python_ast::helpers::{create_expr, unparse_expr};
use ruff_python_ast::source_code::Stylist;
use ruff_python_ast::token_kind::TokenKind;

pub fn is_ambiguous_name(name: &str) -> bool {
name == "l" || name == "I" || name == "O"
Expand Down Expand Up @@ -59,168 +60,37 @@ pub fn is_overlong(
}

pub const fn is_keyword_token(token: &Tok) -> bool {
matches!(
token,
Tok::False
| Tok::True
| Tok::None
| Tok::And
| Tok::As
| Tok::Assert
| Tok::Await
| Tok::Break
| Tok::Class
| Tok::Continue
| Tok::Def
| Tok::Del
| Tok::Elif
| Tok::Else
| Tok::Except
| Tok::Finally
| Tok::For
| Tok::From
| Tok::Global
| Tok::If
| Tok::Import
| Tok::In
| Tok::Is
| Tok::Lambda
| Tok::Nonlocal
| Tok::Not
| Tok::Or
| Tok::Pass
| Tok::Raise
| Tok::Return
| Tok::Try
| Tok::While
| Tok::With
| Tok::Yield
)
TokenKind::from_token(token).is_keyword()
}

pub const fn is_singleton_token(token: &Tok) -> bool {
matches!(
token,
Tok::False { .. } | Tok::True { .. } | Tok::None { .. },
)
TokenKind::from_token(token).is_singleton()
}

pub const fn is_op_token(token: &Tok) -> bool {
matches!(
token,
Tok::Lpar
| Tok::Rpar
| Tok::Lsqb
| Tok::Rsqb
| Tok::Comma
| Tok::Semi
| Tok::Plus
| Tok::Minus
| Tok::Star
| Tok::Slash
| Tok::Vbar
| Tok::Amper
| Tok::Less
| Tok::Greater
| Tok::Equal
| Tok::Dot
| Tok::Percent
| Tok::Lbrace
| Tok::Rbrace
| Tok::NotEqual
| Tok::LessEqual
| Tok::GreaterEqual
| Tok::Tilde
| Tok::CircumFlex
| Tok::LeftShift
| Tok::RightShift
| Tok::DoubleStar
| Tok::PlusEqual
| Tok::MinusEqual
| Tok::StarEqual
| Tok::SlashEqual
| Tok::PercentEqual
| Tok::AmperEqual
| Tok::VbarEqual
| Tok::CircumflexEqual
| Tok::LeftShiftEqual
| Tok::RightShiftEqual
| Tok::DoubleStarEqual
| Tok::DoubleSlash
| Tok::DoubleSlashEqual
| Tok::At
| Tok::AtEqual
| Tok::Rarrow
| Tok::Ellipsis
| Tok::ColonEqual
| Tok::Colon
)
TokenKind::from_token(token).is_operator()
}

pub const fn is_skip_comment_token(token: &Tok) -> bool {
matches!(
token,
Tok::Newline | Tok::Indent | Tok::Dedent | Tok::NonLogicalNewline | Tok::Comment { .. }
)
TokenKind::from_token(token).is_skip_comment()
}

pub const fn is_soft_keyword_token(token: &Tok) -> bool {
matches!(token, Tok::Match | Tok::Case)
TokenKind::from_token(token).is_soft_keyword()
}

pub const fn is_arithmetic_token(token: &Tok) -> bool {
matches!(
token,
Tok::DoubleStar | Tok::Star | Tok::Plus | Tok::Minus | Tok::Slash | Tok::At
)
TokenKind::from_token(token).is_arithmetic()
}

pub const fn is_ws_optional_token(token: &Tok) -> bool {
is_arithmetic_token(token)
|| matches!(
token,
Tok::CircumFlex
| Tok::Amper
| Tok::Vbar
| Tok::LeftShift
| Tok::RightShift
| Tok::Percent
)
TokenKind::from_token(token).is_whitespace_optional()
}

pub const fn is_ws_needed_token(token: &Tok) -> bool {
matches!(
token,
Tok::DoubleStarEqual
| Tok::StarEqual
| Tok::SlashEqual
| Tok::DoubleSlashEqual
| Tok::PlusEqual
| Tok::MinusEqual
| Tok::NotEqual
| Tok::Less
| Tok::Greater
| Tok::PercentEqual
| Tok::CircumflexEqual
| Tok::AmperEqual
| Tok::VbarEqual
| Tok::EqEqual
| Tok::LessEqual
| Tok::GreaterEqual
| Tok::LeftShiftEqual
| Tok::RightShiftEqual
| Tok::Equal
| Tok::And
| Tok::Or
| Tok::In
| Tok::Is
| Tok::Rarrow
)
TokenKind::from_token(token).is_whitespace_needed()
}

pub const fn is_unary_token(token: &Tok) -> bool {
matches!(
token,
Tok::Plus | Tok::Minus | Tok::Star | Tok::DoubleStar | Tok::RightShift
)
TokenKind::from_token(token).is_unary()
}
Loading

0 comments on commit 4224bfe

Please sign in to comment.