From ec46f37b82b375ae184d8d2aef81222999235a4a Mon Sep 17 00:00:00 2001 From: Victorien Elvinger Date: Fri, 1 Nov 2024 08:00:10 +0100 Subject: [PATCH] chore: remove JS parser test generation codegen (#4443) --- Cargo.lock | 1 - xtask/codegen/Cargo.toml | 1 - xtask/codegen/src/lib.rs | 5 - xtask/codegen/src/main.rs | 6 +- xtask/codegen/src/parser_tests.rs | 259 ------------------------------ 5 files changed, 1 insertion(+), 271 deletions(-) delete mode 100644 xtask/codegen/src/parser_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 013bddb73731..085264daa216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4521,7 +4521,6 @@ dependencies = [ "serde", "serde_json", "ureq", - "walkdir", "xtask", ] diff --git a/xtask/codegen/Cargo.toml b/xtask/codegen/Cargo.toml index f948aeac2e7d..610f56351d1f 100644 --- a/xtask/codegen/Cargo.toml +++ b/xtask/codegen/Cargo.toml @@ -13,7 +13,6 @@ pulldown-cmark = { version = "0.12.2", default-features = false, optional = true quote = "1.0.36" serde = { workspace = true, optional = true } ureq = "2.10.1" -walkdir = "2.5.0" xtask = { path = '../', version = "0.0" } biome_analyze = { workspace = true, optional = true } diff --git a/xtask/codegen/src/lib.rs b/xtask/codegen/src/lib.rs index 88eaade701cc..1fee10c886e4 100644 --- a/xtask/codegen/src/lib.rs +++ b/xtask/codegen/src/lib.rs @@ -23,7 +23,6 @@ mod generate_crate; mod html_kinds_src; mod kind_src; mod language_kind; -mod parser_tests; pub mod promote_rule; mod termcolorful; mod unicode; @@ -39,7 +38,6 @@ pub use self::formatter::generate_formatters; pub use self::generate_analyzer::generate_analyzer; pub use self::generate_crate::generate_crate; pub use self::generate_new_analyzer_rule::{generate_new_analyzer_rule, LanguageKind}; -pub use self::parser_tests::generate_parser_tests; pub use self::unicode::generate_tables; pub enum UpdateResult { @@ -105,9 +103,6 @@ pub enum TaskCommand { /// Transforms ungram files into AST #[bpaf(command)] Grammar(Vec), - /// Extracts parser inline comments into test files - #[bpaf(command)] - Test, /// Generates unicode table inside lexer #[bpaf(command)] Unicode, diff --git a/xtask/codegen/src/main.rs b/xtask/codegen/src/main.rs index a8be913bb818..eca4ffa4bb8e 100644 --- a/xtask/codegen/src/main.rs +++ b/xtask/codegen/src/main.rs @@ -26,7 +26,7 @@ use crate::promote_rule::promote_rule; use xtask::Mode::Overwrite; use xtask_codegen::{ generate_analyzer, generate_ast, generate_crate, generate_formatters, - generate_new_analyzer_rule, generate_parser_tests, generate_tables, task_command, TaskCommand, + generate_new_analyzer_rule, generate_tables, task_command, TaskCommand, }; fn main() -> Result<()> { @@ -63,9 +63,6 @@ fn main() -> Result<()> { TaskCommand::Grammar(language_list) => { generate_ast(Overwrite, language_list)?; } - TaskCommand::Test => { - generate_parser_tests(Overwrite)?; - } TaskCommand::Unicode => { generate_tables()?; } @@ -82,7 +79,6 @@ fn main() -> Result<()> { TaskCommand::All => { generate_tables()?; generate_ast(Overwrite, vec![])?; - generate_parser_tests(Overwrite)?; generate_formatters(); generate_analyzer()?; #[cfg(feature = "configuration")] diff --git a/xtask/codegen/src/parser_tests.rs b/xtask/codegen/src/parser_tests.rs deleted file mode 100644 index 50c2ccdbf790..000000000000 --- a/xtask/codegen/src/parser_tests.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Takes comments from biome_js_parser and turns them into test data. -//! This code is derived from rust_analyzer/xtask/codegen/gen_parser_tests - -use std::{ - collections::HashMap, - fs, mem, - path::{Path, PathBuf}, - time::SystemTime, -}; - -use crate::{update, Mode}; -use xtask::{project_root, Result}; - -fn extract_comment_blocks( - text: &str, - allow_blocks_with_empty_lines: bool, -) -> Vec<(usize, Vec)> { - let mut res = Vec::new(); - - let prefix = "// "; - let lines = text.lines().map(str::trim_start); - - let mut block = (0, vec![]); - for (line_num, line) in lines.enumerate() { - if line == "//" && allow_blocks_with_empty_lines { - block.1.push(String::new()); - continue; - } - - let is_comment = line.starts_with(prefix); - if is_comment { - block.1.push(line[prefix.len()..].to_string()); - } else { - if !block.1.is_empty() { - res.push(mem::take(&mut block)); - } - block.0 = line_num + 2; - } - } - if !block.1.is_empty() { - res.push(block) - } - res -} - -pub fn generate_parser_tests(mode: Mode) -> Result<()> { - let tests = tests_from_dir(&project_root().join(Path::new("crates/biome_js_parser/src")))?; - fn install_tests(tests: &HashMap, into: &str, mode: Mode) -> Result { - let tests_dir = project_root().join(into); - if !tests_dir.is_dir() { - fs::create_dir_all(&tests_dir)?; - } - // ok is never actually read, but it needs to be specified to create a Test in existing_tests - let existing = existing_tests(&tests_dir, true)?; - - if let Some(t) = existing.keys().find(|&t| !tests.contains_key(t)) { - panic!("Test is deleted: '{t}'"); - } - - let mut some_file_was_updated = false; - - for (name, test) in tests { - let path = match existing.get(name) { - Some((path, _test)) => path.clone(), - None => tests_dir - .join(name) - .with_extension(test.language.extension()), - }; - if let crate::UpdateResult::Updated = update(&path, &test.text, &mode)? { - some_file_was_updated = true; - } - - if let Some(options) = &test.options { - let path = tests_dir.join(name).with_extension("options.json"); - - if let crate::UpdateResult::Updated = update(&path, options, &mode)? { - some_file_was_updated = true; - } - } - } - - Ok(some_file_was_updated) - } - - let mut some_file_was_updated = false; - some_file_was_updated |= install_tests( - &tests.ok, - "crates/biome_js_parser/test_data/inline/ok", - mode, - )?; - some_file_was_updated |= install_tests( - &tests.err, - "crates/biome_js_parser/test_data/inline/err", - mode, - )?; - - if some_file_was_updated { - fs::File::open("crates/biome_js_parser/src/tests.rs")?.set_modified(SystemTime::now())?; - } - - Ok(()) -} - -#[derive(Debug)] -struct Test { - pub name: String, - pub text: String, - pub ok: bool, - pub language: Language, - pub options: Option, -} - -#[derive(Debug)] -enum Language { - JavaScript, - TypeScript, - TypeScriptDefinition, - Jsx, - Tsx, -} - -impl Language { - const fn extension(&self) -> &'static str { - match self { - Language::JavaScript => "js", - Language::TypeScript => "ts", - Language::TypeScriptDefinition => "d.ts", - Language::Jsx => "jsx", - Language::Tsx => "tsx", - } - } - - fn from_file_name(name: &str) -> Option { - let language = match name.rsplit_once('.')? { - (_, "js") => Language::JavaScript, - (rest, "ts") => match rest.rsplit_once('.') { - Some((_, "d")) => Language::TypeScriptDefinition, - _ => Language::TypeScript, - }, - (_, "jsx") => Language::Jsx, - (_, "tsx") => Language::Tsx, - _ => { - return None; - } - }; - - Some(language) - } -} - -#[derive(Default, Debug)] -struct Tests { - pub ok: HashMap, - pub err: HashMap, -} - -fn collect_tests(s: &str) -> Vec { - let mut res = Vec::new(); - for comment_block in extract_comment_blocks(s, false).into_iter().map(|(_, x)| x) { - let first_line = &comment_block[0]; - - let (ok, suffix) = match first_line.split_once(' ') { - Some(("test", suffix)) => (true, suffix), - Some(("test_err", suffix)) => (false, suffix), - _ => continue, - }; - - let (language, suffix) = match suffix.split_once(' ') { - Some(("jsx", suffix)) => (Language::Jsx, suffix), - Some(("js", suffix)) => (Language::JavaScript, suffix), - Some(("ts", suffix)) => (Language::TypeScript, suffix), - Some(("d.ts", suffix)) => (Language::TypeScriptDefinition, suffix), - Some(("tsx", suffix)) => (Language::Tsx, suffix), - Some((_, suffix)) => (Language::JavaScript, suffix), - _ => panic!("wrong test configuration: {suffix:?}"), - }; - - let (name, options) = match suffix.split_once(' ') { - Some((name, options)) => (name, Some(options.to_string())), - _ => (suffix, None), - }; - - let text: String = comment_block[1..] - .iter() - .cloned() - .chain([String::new()]) - .collect::>() - .join("\n"); - - assert!(!text.trim().is_empty() && text.ends_with('\n')); - res.push(Test { - name: name.to_string(), - options, - text, - ok, - language, - }) - } - res -} - -fn tests_from_dir(dir: &Path) -> Result { - let mut res = Tests::default(); - for entry in ::walkdir::WalkDir::new(dir) { - let entry = entry.unwrap(); - if !entry.file_type().is_file() { - continue; - } - if entry.path().extension().unwrap_or_default() != "rs" { - continue; - } - process_file(&mut res, entry.path())?; - } - return Ok(res); - fn process_file(res: &mut Tests, path: &Path) -> Result<()> { - let text = fs::read_to_string(path)?; - - for test in collect_tests(&text) { - if test.ok { - if let Some(old_test) = res.ok.insert(test.name.clone(), test) { - anyhow::bail!("Duplicate test: {}", old_test.name); - } - } else if let Some(old_test) = res.err.insert(test.name.clone(), test) { - anyhow::bail!("Duplicate test: {}", old_test.name); - } - } - Ok(()) - } -} - -fn existing_tests(dir: &Path, ok: bool) -> Result> { - let mut res = HashMap::new(); - for file in fs::read_dir(dir)? { - let path = file?.path(); - let language = path - .extension() - .and_then(|ext| ext.to_str()) - .and_then(Language::from_file_name); - - if let Some(language) = language { - let name = path - .file_stem() - .map(|x| x.to_string_lossy().to_string()) - .unwrap(); - let text = fs::read_to_string(&path)?; - let test = Test { - name: name.clone(), - options: None, - text, - ok, - language, - }; - if let Some(old) = res.insert(name, (path, test)) { - println!("Duplicate test: {old:?}"); - } - } - } - Ok(res) -}