Skip to content

Commit

Permalink
Reduce Tok size by using Box<str> instead of String
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaReiser committed Feb 8, 2024
1 parent 1ec90fb commit 4c7142b
Show file tree
Hide file tree
Showing 20 changed files with 260 additions and 248 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option<String> {
}

let node = Expr::from(ast::StringLiteral {
value: elts.iter().fold(String::new(), |mut acc, elt| {
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
if !acc.is_empty() {
acc.push(',');
value: elts
.iter()
.fold(String::new(), |mut acc, elt| {
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
if !acc.is_empty() {
acc.push(',');
}
acc.push_str(value.to_str());
}
acc.push_str(value.to_str());
}
acc
}),
acc
})
.into_boxed_str(),
..ast::StringLiteral::default()
});
Some(generator.expr(&node))
Expand Down Expand Up @@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
.iter()
.map(|name| {
Expr::from(ast::StringLiteral {
value: (*name).to_string(),
value: (*name).to_string().into_boxed_str(),
..ast::StringLiteral::default()
})
})
Expand Down Expand Up @@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
.iter()
.map(|name| {
Expr::from(ast::StringLiteral {
value: (*name).to_string(),
value: (*name).to_string().into_boxed_str(),
..ast::StringLiteral::default()
})
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) {
slice.range(),
);
let node = ast::StringLiteral {
value: capital_env_var,
value: capital_env_var.into_boxed_str(),
unicode: env_var.is_unicode(),
..ast::StringLiteral::default()
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
None
}
})
.join(joiner),
.join(joiner)
.into_boxed_str(),
..ast::StringLiteral::default()
};
return Some(node.into());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence(
let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else {
return;
};
(value.as_str(), range.start())
(value.as_ref(), range.start())
}
Tok::String { kind, .. } => {
if kind.is_raw() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix {
.generator()
.expr(&Expr::StringLiteral(ast::ExprStringLiteral {
value: ast::StringLiteralValue::single(ast::StringLiteral {
value: "locale".to_string(),
value: "locale".to_string().into_boxed_str(),
unicode: false,
range: TextRange::default(),
}),
Expand Down
2 changes: 1 addition & 1 deletion crates/ruff_linter/src/rules/pyupgrade/fixes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String
let last_range = names.last_mut().unwrap();
*last_range = TextRange::new(last_range.start(), range.end());
} else {
if members.contains(&name.as_str()) {
if members.contains(&name.as_ref()) {
removal_indices.push(names.len());
}
names.push(range);
Expand Down
16 changes: 8 additions & 8 deletions crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -559,14 +559,14 @@ fn collect_string_sequence_lines(
/// `self` and produces the classification for the line.
#[derive(Debug, Default)]
struct LineState {
first_item_in_line: Option<(String, TextRange)>,
following_items_in_line: Vec<(String, TextRange)>,
first_item_in_line: Option<(Box<str>, TextRange)>,
following_items_in_line: Vec<(Box<str>, TextRange)>,
comment_range_start: Option<TextSize>,
comment_in_line: Option<TextRange>,
}

impl LineState {
fn visit_string_token(&mut self, token_value: String, token_range: TextRange) {
fn visit_string_token(&mut self, token_value: Box<str>, token_range: TextRange) {
if self.first_item_in_line.is_none() {
self.first_item_in_line = Some((token_value, token_range));
} else {
Expand Down Expand Up @@ -631,8 +631,8 @@ struct LineWithItems {
// For elements in the list, we keep track of the value of the
// value of the element as well as the source-code range of the element.
// (We need to know the actual value so that we can sort the items.)
first_item: (String, TextRange),
following_items: Vec<(String, TextRange)>,
first_item: (Box<str>, TextRange),
following_items: Vec<(Box<str>, TextRange)>,
// For comments, we only need to keep track of the source-code range.
trailing_comment_range: Option<TextRange>,
}
Expand Down Expand Up @@ -753,7 +753,7 @@ fn collect_string_sequence_items(
/// source-code range of `"a"`.
#[derive(Debug)]
struct StringSequenceItem {
value: String,
value: Box<str>,
preceding_comment_ranges: Vec<TextRange>,
element_range: TextRange,
// total_range incorporates the ranges of preceding comments
Expand All @@ -766,7 +766,7 @@ struct StringSequenceItem {

impl StringSequenceItem {
fn new(
value: String,
value: Box<str>,
preceding_comment_ranges: Vec<TextRange>,
element_range: TextRange,
end_of_line_comments: Option<TextRange>,
Expand All @@ -787,7 +787,7 @@ impl StringSequenceItem {
}
}

fn with_no_comments(value: String, element_range: TextRange) -> Self {
fn with_no_comments(value: Box<str>, element_range: TextRange) -> Self {
Self::new(value, vec![], element_range, None)
}
}
Expand Down
12 changes: 3 additions & 9 deletions crates/ruff_python_ast/src/comparable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> {
impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> {
fn from(string_literal: &'a ast::StringLiteral) -> Self {
Self {
value: string_literal.value.as_str(),
value: &string_literal.value,
}
}
}
Expand Down Expand Up @@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
kind,
value,
range: _,
}) => Self::IpyEscapeCommand(ExprIpyEscapeCommand {
kind: *kind,
value: value.as_str(),
}),
}) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }),
}
}
}
Expand Down Expand Up @@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
kind,
value,
range: _,
}) => Self::IpyEscapeCommand(StmtIpyEscapeCommand {
kind: *kind,
value: value.as_str(),
}),
}) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }),
ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr {
value: value.into(),
}),
Expand Down
16 changes: 9 additions & 7 deletions crates/ruff_python_ast/src/nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ pub enum Stmt {
pub struct StmtIpyEscapeCommand {
pub range: TextRange,
pub kind: IpyEscapeKind,
pub value: String,
pub value: Box<str>,
}

impl From<StmtIpyEscapeCommand> for Stmt {
Expand Down Expand Up @@ -671,7 +671,7 @@ impl Expr {
pub struct ExprIpyEscapeCommand {
pub range: TextRange,
pub kind: IpyEscapeKind,
pub value: String,
pub value: Box<str>,
}

impl From<ExprIpyEscapeCommand> for Expr {
Expand Down Expand Up @@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner {
#[derive(Clone, Debug, Default, PartialEq)]
pub struct StringLiteral {
pub range: TextRange,
pub value: String,
pub value: Box<str>,
pub unicode: bool,
}

Expand All @@ -1398,7 +1398,7 @@ impl Deref for StringLiteral {
type Target = str;

fn deref(&self) -> &Self::Target {
self.value.as_str()
&self.value
}
}

Expand Down Expand Up @@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral {
/// Each string literal that makes up the concatenated string.
strings: Vec<StringLiteral>,
/// The concatenated string value.
value: OnceCell<String>,
value: OnceCell<Box<str>>,
}

impl ConcatenatedStringLiteral {
/// Extracts a string slice containing the entire concatenated string.
fn to_str(&self) -> &str {
self.value
.get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect())
self.value.get_or_init(|| {
let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect();
concatenated.into_boxed_str()
})
}
}

Expand Down
12 changes: 8 additions & 4 deletions crates/ruff_python_formatter/tests/normalizer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,19 +95,22 @@ impl Transformer for Normalizer {
&string_literal.value,
"<DOCTEST-CODE-SNIPPET: Removed by normalizer>\n",
)
.into_owned();
.into_owned()
.into_boxed_str();
string_literal.value = STRIP_RST_BLOCKS
.replace_all(
&string_literal.value,
"<RSTBLOCK-CODE-SNIPPET: Removed by normalizer>\n",
)
.into_owned();
.into_owned()
.into_boxed_str();
string_literal.value = STRIP_MARKDOWN_BLOCKS
.replace_all(
&string_literal.value,
"<MARKDOWN-CODE-SNIPPET: Removed by normalizer>\n",
)
.into_owned();
.into_owned()
.into_boxed_str();
// Normalize a string by (2) stripping any leading and trailing space from each
// line, and (3) removing any blank lines from the start and end of the string.
string_literal.value = string_literal
Expand All @@ -117,6 +120,7 @@ impl Transformer for Normalizer {
.collect::<Vec<_>>()
.join("\n")
.trim()
.to_owned();
.to_owned()
.into_boxed_str();
}
}
28 changes: 15 additions & 13 deletions crates/ruff_python_parser/src/function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ pub(crate) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), Lexi
let arg_name = arg.name.as_str();
if !all_arg_names.insert(arg_name) {
return Err(LexicalError::new(
LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()),
range.start(),
));
}
Expand Down Expand Up @@ -95,7 +95,9 @@ pub(crate) fn parse_arguments(
if let Some(keyword_name) = &name {
if !keyword_names.insert(keyword_name.to_string()) {
return Err(LexicalError::new(
LexicalErrorType::DuplicateKeywordArgumentError(keyword_name.to_string()),
LexicalErrorType::DuplicateKeywordArgumentError(
keyword_name.to_string().into_boxed_str(),
),
start,
));
}
Expand Down Expand Up @@ -200,22 +202,22 @@ mod tests {

function_and_lambda_error! {
// Check definitions
test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError,
test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError,

// Check some calls.
test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError,
test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError,
test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()),
test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string().into_boxed_str()),
}
}
Loading

0 comments on commit 4c7142b

Please sign in to comment.