From 69f2c9c101259307c9d278744d32310d7a12dbf8 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 27 Nov 2023 19:23:20 +1100 Subject: [PATCH] Move `gather_comments`. To the module where it is used, so it doesn't have to be `pub`. --- Cargo.lock | 1 + compiler/rustc_ast/src/util/comments.rs | 126 +----------------- compiler/rustc_ast_pretty/Cargo.toml | 1 + compiler/rustc_ast_pretty/src/pprust/state.rs | 125 ++++++++++++++++- 4 files changed, 126 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b74d71fd8929..7be392a8435dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3487,6 +3487,7 @@ version = "0.0.0" dependencies = [ "itertools 0.11.0", "rustc_ast", + "rustc_lexer", "rustc_span", "thin-vec", ] diff --git a/compiler/rustc_ast/src/util/comments.rs b/compiler/rustc_ast/src/util/comments.rs index bdf5143b0f706..cbc1afc6bf1e8 100644 --- a/compiler/rustc_ast/src/util/comments.rs +++ b/compiler/rustc_ast/src/util/comments.rs @@ -1,6 +1,5 @@ use crate::token::CommentKind; -use rustc_span::source_map::SourceMap; -use rustc_span::{BytePos, CharPos, FileName, Pos, Symbol}; +use rustc_span::{BytePos, Symbol}; #[cfg(test)] mod tests; @@ -131,126 +130,3 @@ pub fn beautify_doc_string(data: Symbol, kind: CommentKind) -> Symbol { } data } - -/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. -/// Otherwise returns `Some(k)` where `k` is first char offset after that leading -/// whitespace. Note that `k` may be outside bounds of `s`. -fn all_whitespace(s: &str, col: CharPos) -> Option { - let mut idx = 0; - for (i, ch) in s.char_indices().take(col.to_usize()) { - if !ch.is_whitespace() { - return None; - } - idx = i + ch.len_utf8(); - } - Some(idx) -} - -fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str { - let len = s.len(); - match all_whitespace(s, col) { - Some(col) => { - if col < len { - &s[col..] - } else { - "" - } - } - None => s, - } -} - -fn split_block_comment_into_lines(text: &str, col: CharPos) -> Vec { - let mut res: Vec = vec![]; - let mut lines = text.lines(); - // just push the first line - res.extend(lines.next().map(|it| it.to_string())); - // for other lines, strip common whitespace prefix - for line in lines { - res.push(trim_whitespace_prefix(line, col).to_string()) - } - res -} - -// it appears this function is called only from pprust... that's -// probably not a good thing. -pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec { - let sm = SourceMap::new(sm.path_mapping().clone()); - let source_file = sm.new_source_file(path, src); - let text = (*source_file.src.as_ref().unwrap()).clone(); - - let text: &str = text.as_str(); - let start_bpos = source_file.start_pos; - let mut pos = 0; - let mut comments: Vec = Vec::new(); - let mut code_to_the_left = false; - - if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { - comments.push(Comment { - style: CommentStyle::Isolated, - lines: vec![text[..shebang_len].to_string()], - pos: start_bpos, - }); - pos += shebang_len; - } - - for token in rustc_lexer::tokenize(&text[pos..]) { - let token_text = &text[pos..pos + token.len as usize]; - match token.kind { - rustc_lexer::TokenKind::Whitespace => { - if let Some(mut idx) = token_text.find('\n') { - code_to_the_left = false; - while let Some(next_newline) = &token_text[idx + 1..].find('\n') { - idx += 1 + next_newline; - comments.push(Comment { - style: CommentStyle::BlankLine, - lines: vec![], - pos: start_bpos + BytePos((pos + idx) as u32), - }); - } - } - } - rustc_lexer::TokenKind::BlockComment { doc_style, .. } => { - if doc_style.is_none() { - let code_to_the_right = !matches!( - text[pos + token.len as usize..].chars().next(), - Some('\r' | '\n') - ); - let style = match (code_to_the_left, code_to_the_right) { - (_, true) => CommentStyle::Mixed, - (false, false) => CommentStyle::Isolated, - (true, false) => CommentStyle::Trailing, - }; - - // Count the number of chars since the start of the line by rescanning. - let pos_in_file = start_bpos + BytePos(pos as u32); - let line_begin_in_file = source_file.line_begin_pos(pos_in_file); - let line_begin_pos = (line_begin_in_file - start_bpos).to_usize(); - let col = CharPos(text[line_begin_pos..pos].chars().count()); - - let lines = split_block_comment_into_lines(token_text, col); - comments.push(Comment { style, lines, pos: pos_in_file }) - } - } - rustc_lexer::TokenKind::LineComment { doc_style } => { - if doc_style.is_none() { - comments.push(Comment { - style: if code_to_the_left { - CommentStyle::Trailing - } else { - CommentStyle::Isolated - }, - lines: vec![token_text.to_string()], - pos: start_bpos + BytePos(pos as u32), - }) - } - } - _ => { - code_to_the_left = true; - } - } - pos += token.len as usize; - } - - comments -} diff --git a/compiler/rustc_ast_pretty/Cargo.toml b/compiler/rustc_ast_pretty/Cargo.toml index 12a08f06558ba..b38a2915a43df 100644 --- a/compiler/rustc_ast_pretty/Cargo.toml +++ b/compiler/rustc_ast_pretty/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" # tidy-alphabetical-start itertools = "0.11" rustc_ast = { path = "../rustc_ast" } +rustc_lexer = { path = "../rustc_lexer" } rustc_span = { path = "../rustc_span" } thin-vec = "0.2.12" # tidy-alphabetical-end diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 6e1974f48b26b..6119c6c84f8b7 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -14,7 +14,7 @@ use rustc_ast::ptr::P; use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree}; use rustc_ast::util::classify; -use rustc_ast::util::comments::{gather_comments, Comment, CommentStyle}; +use rustc_ast::util::comments::{Comment, CommentStyle}; use rustc_ast::util::parser; use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, BlockCheckMode, PatKind}; use rustc_ast::{attr, BindingAnnotation, ByRef, DelimArgs, RangeEnd, RangeSyntax, Term}; @@ -24,7 +24,7 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_span::edition::Edition; use rustc_span::source_map::{SourceMap, Spanned}; use rustc_span::symbol::{kw, sym, Ident, IdentPrinter, Symbol}; -use rustc_span::{BytePos, FileName, Span, DUMMY_SP}; +use rustc_span::{BytePos, CharPos, FileName, Pos, Span, DUMMY_SP}; use std::borrow::Cow; use thin_vec::ThinVec; @@ -59,6 +59,127 @@ pub struct Comments<'a> { current: usize, } +/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. +/// Otherwise returns `Some(k)` where `k` is first char offset after that leading +/// whitespace. Note that `k` may be outside bounds of `s`. +fn all_whitespace(s: &str, col: CharPos) -> Option { + let mut idx = 0; + for (i, ch) in s.char_indices().take(col.to_usize()) { + if !ch.is_whitespace() { + return None; + } + idx = i + ch.len_utf8(); + } + Some(idx) +} + +fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str { + let len = s.len(); + match all_whitespace(s, col) { + Some(col) => { + if col < len { + &s[col..] + } else { + "" + } + } + None => s, + } +} + +fn split_block_comment_into_lines(text: &str, col: CharPos) -> Vec { + let mut res: Vec = vec![]; + let mut lines = text.lines(); + // just push the first line + res.extend(lines.next().map(|it| it.to_string())); + // for other lines, strip common whitespace prefix + for line in lines { + res.push(trim_whitespace_prefix(line, col).to_string()) + } + res +} + +fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec { + let sm = SourceMap::new(sm.path_mapping().clone()); + let source_file = sm.new_source_file(path, src); + let text = (*source_file.src.as_ref().unwrap()).clone(); + + let text: &str = text.as_str(); + let start_bpos = source_file.start_pos; + let mut pos = 0; + let mut comments: Vec = Vec::new(); + let mut code_to_the_left = false; + + if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { + comments.push(Comment { + style: CommentStyle::Isolated, + lines: vec![text[..shebang_len].to_string()], + pos: start_bpos, + }); + pos += shebang_len; + } + + for token in rustc_lexer::tokenize(&text[pos..]) { + let token_text = &text[pos..pos + token.len as usize]; + match token.kind { + rustc_lexer::TokenKind::Whitespace => { + if let Some(mut idx) = token_text.find('\n') { + code_to_the_left = false; + while let Some(next_newline) = &token_text[idx + 1..].find('\n') { + idx += 1 + next_newline; + comments.push(Comment { + style: CommentStyle::BlankLine, + lines: vec![], + pos: start_bpos + BytePos((pos + idx) as u32), + }); + } + } + } + rustc_lexer::TokenKind::BlockComment { doc_style, .. } => { + if doc_style.is_none() { + let code_to_the_right = !matches!( + text[pos + token.len as usize..].chars().next(), + Some('\r' | '\n') + ); + let style = match (code_to_the_left, code_to_the_right) { + (_, true) => CommentStyle::Mixed, + (false, false) => CommentStyle::Isolated, + (true, false) => CommentStyle::Trailing, + }; + + // Count the number of chars since the start of the line by rescanning. + let pos_in_file = start_bpos + BytePos(pos as u32); + let line_begin_in_file = source_file.line_begin_pos(pos_in_file); + let line_begin_pos = (line_begin_in_file - start_bpos).to_usize(); + let col = CharPos(text[line_begin_pos..pos].chars().count()); + + let lines = split_block_comment_into_lines(token_text, col); + comments.push(Comment { style, lines, pos: pos_in_file }) + } + } + rustc_lexer::TokenKind::LineComment { doc_style } => { + if doc_style.is_none() { + comments.push(Comment { + style: if code_to_the_left { + CommentStyle::Trailing + } else { + CommentStyle::Isolated + }, + lines: vec![token_text.to_string()], + pos: start_bpos + BytePos(pos as u32), + }) + } + } + _ => { + code_to_the_left = true; + } + } + pos += token.len as usize; + } + + comments +} + impl<'a> Comments<'a> { pub fn new(sm: &'a SourceMap, filename: FileName, input: String) -> Comments<'a> { let comments = gather_comments(sm, filename, input);