diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..c41542feb22 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test262"] + path = test262 + url = https://github.com/tc39/test262.git diff --git a/Cargo.lock b/Cargo.lock index ceb3a671aff..660033413ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,6 +178,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -310,6 +321,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + [[package]] name = "either" version = "1.5.3" @@ -322,6 +339,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "gc" version = "0.3.6" @@ -410,9 +436,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.41" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" +checksum = "52732a3d3ad72c58ad2dc70624f9c17b46ecd0943b9a4f1ee37c4c18c5d983e2" dependencies = [ "wasm-bindgen", ] @@ -429,6 +455,12 @@ version = "0.2.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + [[package]] name = "lock_api" version = "0.3.4" @@ -440,9 +472,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ "cfg-if", ] @@ -867,6 +899,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3e2dd40a7cdc18ca80db804b7f461a39bb721160a85c9a1fa30134bf3c02a5" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + [[package]] name = "smallvec" version = "0.6.13" @@ -940,6 +984,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tester" +version = "0.1.0" +dependencies = [ + "Boa", + "bitflags", + "colored", + "fxhash", + "once_cell", + "regex", + "serde", + "serde_yaml", + "structopt", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1029,9 +1088,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.64" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" +checksum = "f3edbcc9536ab7eababcc6d2374a0b7bfe13a2b6d562c5e07f370456b1a8f33d" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1039,9 +1098,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.64" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" +checksum = "89ed2fb8c84bfad20ea66b26a3743f3e7ba8735a69fe7d95118c33ec8fc1244d" dependencies = [ "bumpalo", "lazy_static", @@ -1054,9 +1113,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.64" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" +checksum = "eb071268b031a64d92fc6cf691715ca5a40950694d8f683c5bb43db7c730929e" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1064,9 +1123,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.64" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" +checksum = "cf592c807080719d1ff2f245a687cbadb3ed28b2077ed7084b47aba8b691f2c6" dependencies = [ "proc-macro2", "quote", @@ -1077,15 +1136,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.64" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" +checksum = "72b6c0220ded549d63860c78c38f3bcc558d1ca3f4efa74942c536ddbbb55e87" [[package]] name = "web-sys" -version = "0.3.41" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" +checksum = "8be2398f326b7ba09815d0b403095f34dd708579220d099caae89be0b32137b2" dependencies = [ "js-sys", "wasm-bindgen", @@ -1121,3 +1180,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" +dependencies = [ + "linked-hash-map", +] diff --git a/Cargo.toml b/Cargo.toml index 5c1ef90c6f2..a3b2b1f608b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "boa", "boa_cli", "boa_wasm", + "tester", ] # The release profile, used for `cargo build --release`. diff --git a/boa/src/builtins/string/mod.rs b/boa/src/builtins/string/mod.rs index fa0e00eaf92..9980b57042d 100644 --- a/boa/src/builtins/string/mod.rs +++ b/boa/src/builtins/string/mod.rs @@ -497,7 +497,7 @@ impl String { // Push the whole string being examined results.push(Value::from(primitive_val.to_string())); - let result = ctx.call(&replace_object, this, &results).unwrap(); + let result = ctx.call(&replace_object, this, &results)?; ctx.to_string(&result)?.to_string() } diff --git a/boa/src/exec/call/mod.rs b/boa/src/exec/call/mod.rs index 363c165327a..facc8f070a0 100644 --- a/boa/src/exec/call/mod.rs +++ b/boa/src/exec/call/mod.rs @@ -11,10 +11,8 @@ impl Executable for Call { let (this, func) = match self.expr() { Node::GetConstField(ref get_const_field) => { let mut obj = get_const_field.obj().run(interpreter)?; - if obj.get_type() != Type::Object || obj.get_type() != Type::Symbol { - obj = interpreter - .to_object(&obj) - .expect("failed to convert to object"); + if obj.get_type() != Type::Object { + obj = interpreter.to_object(&obj)?; } (obj.clone(), obj.get_field(get_const_field.field())) } diff --git a/boa/src/exec/field/mod.rs b/boa/src/exec/field/mod.rs index 9e3344c9265..4dc1e7f9a68 100644 --- a/boa/src/exec/field/mod.rs +++ b/boa/src/exec/field/mod.rs @@ -7,10 +7,9 @@ use crate::{ impl Executable for GetConstField { fn run(&self, interpreter: &mut Interpreter) -> ResultValue { let mut obj = self.obj().run(interpreter)?; - if obj.get_type() != Type::Object || obj.get_type() != Type::Symbol { + if obj.get_type() != Type::Object { obj = interpreter.to_object(&obj)?; } - Ok(obj.get_field(self.field())) } } @@ -18,7 +17,7 @@ impl Executable for GetConstField { impl Executable for GetField { fn run(&self, interpreter: &mut Interpreter) -> ResultValue { let mut obj = self.obj().run(interpreter)?; - if obj.get_type() != Type::Object || obj.get_type() != Type::Symbol { + if obj.get_type() != Type::Object { obj = interpreter.to_object(&obj)?; } let field = self.field().run(interpreter)?; diff --git a/boa/src/lib.rs b/boa/src/lib.rs index 37da335d27c..296e06b1b16 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -46,25 +46,32 @@ pub use crate::{ exec::{Executable, Interpreter}, profiler::BoaProfiler, realm::Realm, - syntax::{lexer::Lexer, parser::Parser}, + syntax::{ + lexer::Lexer, + parser::{error::ParseError, Parser}, + }, }; -fn parser_expr(src: &str) -> Result { - Parser::new(src.as_bytes()) - .parse_all() - .map_err(|e| format!("Parsing Error: {}", e)) +/// Parses the given source code. +/// +/// It will return either the statement list AST node for the code, or a parsing error if something +/// goes wrong. +#[inline] +pub fn parse(src: &str) -> Result { + Parser::new(src.as_bytes()).parse_all() } /// Execute the code using an existing Interpreter +/// /// The str is consumed and the state of the Interpreter is changed pub fn forward(engine: &mut Interpreter, src: &str) -> String { // Setup executor - let expr = match parser_expr(src) { + let expr = match parse(src) { Ok(res) => res, - Err(e) => return e, + Err(e) => return format!("Uncaught {}", e), }; expr.run(engine) - .map_or_else(|e| format!("Error: {}", e), |v| v.to_string()) + .map_or_else(|e| format!("Uncaught {}", e), |v| v.to_string()) } /// Execute the code using an existing Interpreter. @@ -75,13 +82,13 @@ pub fn forward(engine: &mut Interpreter, src: &str) -> String { pub fn forward_val(engine: &mut Interpreter, src: &str) -> ResultValue { let main_timer = BoaProfiler::global().start_event("Main", "Main"); // Setup executor - let result = match parser_expr(src) { - Ok(expr) => expr.run(engine), - Err(e) => { - eprintln!("{}", e); - std::process::exit(1); - } - }; + let result = parse(src) + .map_err(|e| { + engine + .throw_syntax_error(e.to_string()) + .expect_err("interpreter.throw_syntax_error() did not return an error") + }) + .and_then(|expr| expr.run(engine)); // The main_timer needs to be dropped before the BoaProfiler is. drop(main_timer); diff --git a/boa/src/syntax/parser/expression/mod.rs b/boa/src/syntax/parser/expression/mod.rs index 66634e11bd4..87498ba20be 100644 --- a/boa/src/syntax/parser/expression/mod.rs +++ b/boa/src/syntax/parser/expression/mod.rs @@ -74,7 +74,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo TokenKind::Punctuator(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( - op.as_binop().expect("Could not get binary operation."), + op.as_binop().unwrap_or_else(|| panic!("could not get binary operation for {:?}", op)), lhs, $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); @@ -82,7 +82,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo TokenKind::Keyword(op) if $( op == $op )||* => { let _ = cursor.next().expect("token disappeared"); lhs = BinOp::new( - op.as_binop().expect("Could not get binary operation."), + op.as_binop().unwrap_or_else(|| panic!("could not get binary operation for {:?}", op)), lhs, $lower::new($( self.$low_param ),*).parse(cursor)? ).into(); diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 7221eada954..db8ebc73fe6 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -70,6 +70,7 @@ where fn parse(self, cursor: &mut Cursor) -> ParseResult { let _timer = BoaProfiler::global().start_event("PrimaryExpression", "Parsing"); + cursor.skip_line_terminators()?; let tok = cursor.next()?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { diff --git a/boa_cli/Cargo.toml b/boa_cli/Cargo.toml index bf9dd0db125..09c1c4c1e81 100644 --- a/boa_cli/Cargo.toml +++ b/boa_cli/Cargo.toml @@ -9,6 +9,7 @@ categories = ["command-line-utilities"] license = "Unlicense/MIT" exclude = ["../.vscode/*", "../Dockerfile", "../Makefile", "../.editorConfig"] edition = "2018" +default-run = "boa" [dependencies] Boa = { path = "../boa", features = ["serde"] } diff --git a/test262 b/test262 new file mode 160000 index 00000000000..28ec03f4542 --- /dev/null +++ b/test262 @@ -0,0 +1 @@ +Subproject commit 28ec03f45428b3b4a4c96445372c9e89c56a2176 diff --git a/test_ignore.txt b/test_ignore.txt new file mode 100644 index 00000000000..cb641741a26 --- /dev/null +++ b/test_ignore.txt @@ -0,0 +1,41 @@ +// This does not break the tester but it does iterate from 0 to u32::MAX, +// because of incorect implementation of `Array.prototype.indexOf`. +// TODO: Fix it do iterate on the elements in the array **in insertion order**, not from +// 0 to u32::MAX untill it reaches the element. +15.4.4.14-5-12 +// Another one of these `Array.prototype.indexOf`, but now with `NaN`. +15.4.4.14-5-14 +// Another one of these `Array.prototype.indexOf`, but now with `-Infinity`. +15.4.4.14-5-13 +// More `Array.prototype.indexOf` with large second argument. +// Others: +15.4.4.14-9-9 +fill-string-empty +// Stack overflow (seemingly in JSON.stringify with circular references): +value-array-circular +value-object-circular +// other +var-env-var-init-global-new +var-env-func-init-global-update-configurable +var-env-var-init-global-exstng +var-env-func-init-global-new +var-env-func-init-global-update-non-configurable +acquire-properties-from-object +acquire-properties-from-array +nonconfigurable-nonenumerable-nonwritable-descriptors-basic +nonwritable-nonenumerable-nonconfigurable-descriptors-set-by-param +nonconfigurable-nonenumerable-nonwritable-descriptors-set-by-param +nonwritable-nonconfigurable-descriptors-set-by-param +nonwritable-nonconfigurable-descriptors-basic +nonconfigurable-nonwritable-descriptors-basic +nonconfigurable-nonenumerable-nonwritable-descriptors-set-by-arguments +nonconfigurable-descriptors-basic +nonconfigurable-nonwritable-descriptors-set-by-param +Symbol.iterator +nonconfigurable-descriptors-set-value-with-define-property +nonconfigurable-nonwritable-descriptors-set-by-arguments +nonwritable-nonconfigurable-descriptors-set-by-arguments +nonconfigurable-descriptors-set-value-by-arguments +nonwritable-nonenumerable-nonconfigurable-descriptors-set-by-arguments +nonwritable-nonenumerable-nonconfigurable-descriptors-set-by-define-property +nonconfigurable-descriptors-with-param-assign diff --git a/tester/Cargo.toml b/tester/Cargo.toml new file mode 100644 index 00000000000..8aa722b17d7 --- /dev/null +++ b/tester/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "tester" +version = "0.1.0" +authors = ["Iban Eguia Moraza "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +Boa = { path = "../boa" } +structopt = "0.3.15" +serde = "1.0.114" +serde_yaml = "0.8.13" +bitflags = "1.2.1" +regex = "1.3.9" +once_cell = "1.4.0" +colored = "2.0.0" +fxhash = "0.2.1" diff --git a/tester/src/exec.rs b/tester/src/exec.rs new file mode 100644 index 00000000000..67a1329402e --- /dev/null +++ b/tester/src/exec.rs @@ -0,0 +1,185 @@ +//! Execution module for the test runner. + +use super::{Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestResult, TestSuite}; +use boa::{forward_val, parse, Interpreter, Realm}; +use colored::Colorize; +use fxhash::FxHashSet; +use once_cell::sync::Lazy; +use std::{fs, panic, path::Path}; + +/// List of ignored tests. +static IGNORED: Lazy>> = Lazy::new(|| { + let path = Path::new("test_ignore.txt"); + if path.exists() { + let filtered = fs::read_to_string(path).expect("could not read test filters"); + filtered + .lines() + .filter(|line| !line.is_empty() && !line.starts_with("//")) + .map(|line| line.to_owned().into_boxed_str()) + .collect::>() + } else { + FxHashSet::default() + } +}); + +impl TestSuite { + /// Runs the test suite. + pub(crate) fn run(&self, harness: &Harness) -> SuiteResult { + println!("Suite {}:", self.name); + + // TODO: in parallel + let suites: Vec<_> = self.suites.iter().map(|suite| suite.run(harness)).collect(); + + // TODO: in parallel + let tests: Vec<_> = self.tests.iter().map(|test| test.run(harness)).collect(); + + println!(); + + // Count passed tests + let mut passed_tests = 0; + let mut ignored_tests = 0; + for test in &tests { + if let Some(true) = test.passed { + passed_tests += 1; + } else if test.passed.is_none() { + ignored_tests += 1; + } + } + + // Count total tests + let mut total_tests = tests.len(); + for suite in &suites { + total_tests += suite.total_tests; + passed_tests += suite.passed_tests; + ignored_tests += suite.ignored_tests; + } + + let passed = passed_tests == total_tests; + + println!( + "Results: total: {}, passed: {}, ignored: {}, conformance: {:.2}%", + total_tests, + passed_tests, + ignored_tests, + (passed_tests as f64 / total_tests as f64) * 100.0 + ); + + SuiteResult { + name: self.name.clone(), + passed, + total_tests, + passed_tests, + ignored_tests, + suites: suites.into_boxed_slice(), + tests: tests.into_boxed_slice(), + } + } +} + +impl Test { + /// Runs the test. + pub(crate) fn run(&self, harness: &Harness) -> TestResult { + println!("Starting `{}`", self.name); + + let passed = if !self.flags.intersects(TestFlags::ASYNC | TestFlags::MODULE) + && !IGNORED.contains(&self.name) + { + let res = panic::catch_unwind(|| { + match self.expected_outcome { + Outcome::Positive => { + let mut passed = true; + + if self.flags.contains(TestFlags::RAW) { + let mut engine = self.set_up_env(&harness, false); + let res = forward_val(&mut engine, &self.content); + + passed = res.is_ok() + } else { + if self.flags.contains(TestFlags::STRICT) { + let mut engine = self.set_up_env(&harness, true); + let res = forward_val(&mut engine, &self.content); + + passed = res.is_ok() + } + + if passed && self.flags.contains(TestFlags::NO_STRICT) { + let mut engine = self.set_up_env(&harness, false); + let res = forward_val(&mut engine, &self.content); + + passed = res.is_ok() + } + } + + passed + } + Outcome::Negative { + phase: Phase::Parse, + ref error_type, + } => { + assert_eq!( + error_type.as_ref(), + "SyntaxError", + "non-SyntaxError parsing error found in {}", + self.name + ); + + parse(&self.content).is_err() + } + Outcome::Negative { + phase, + ref error_type, + } => { + // TODO: check the phase + false + } + } + }); + + let passed = res.unwrap_or(false); + + print!("{}", if passed { ".".green() } else { ".".red() }); + + Some(passed) + } else { + // Ignoring async tests for now. + // TODO: implement async and add `harness/doneprintHandle.js` to the includes. + print!("{}", ".".yellow()); + None + }; + + TestResult { + name: self.name.clone(), + passed, + } + } + + /// Sets the environment up to run the test. + fn set_up_env(&self, harness: &Harness, strict: bool) -> Interpreter { + // Create new Realm + // TODO: in parallel. + let realm = Realm::create(); + let mut engine = Interpreter::new(realm); + + // TODO: set up the environment. + + if strict { + forward_val(&mut engine, r#""use strict";"#).expect("could not set strict mode"); + } + + forward_val(&mut engine, &harness.assert).expect("could not run assert.js"); + forward_val(&mut engine, &harness.sta).expect("could not run sta.js"); + + self.includes.iter().for_each(|include| { + forward_val( + &mut engine, + &harness + .includes + .get(include) + .expect("could not find include file"), + ) + .expect("could not run the include file"); + }); + + engine + } +} diff --git a/tester/src/main.rs b/tester/src/main.rs new file mode 100644 index 00000000000..7ff57a629f7 --- /dev/null +++ b/tester/src/main.rs @@ -0,0 +1,257 @@ +//! Test262 test runner +//! +//! This crate will run the full ECMAScript test suite (Test262) and report compliance of the +//! `boa` engine. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/jasonwilliams/boa/master/assets/logo.svg", + html_favicon_url = "https://raw.githubusercontent.com/jasonwilliams/boa/master/assets/logo.svg" +)] +#![deny( + unused_qualifications, + clippy::all, + unused_qualifications, + unused_import_braces, + unused_lifetimes, + unreachable_pub, + trivial_numeric_casts, + // rustdoc, + missing_debug_implementations, + missing_copy_implementations, + deprecated_in_future, + meta_variable_misuse, + non_ascii_idents, + rust_2018_compatibility, + rust_2018_idioms, + future_incompatible, + nonstandard_style, +)] +#![warn(clippy::perf, clippy::single_match_else, clippy::dbg_macro)] +#![allow( + clippy::missing_inline_in_public_items, + clippy::cognitive_complexity, + clippy::must_use_candidate, + clippy::missing_errors_doc, + clippy::as_conversions, + clippy::let_unit_value, + missing_doc_code_examples +)] + +mod exec; +mod read; + +use self::read::{read_global_suite, read_harness, MetaData, Negative, TestFlag}; +use bitflags::bitflags; +use fxhash::FxHashMap; +use serde::Deserialize; +use std::path::PathBuf; +use structopt::StructOpt; + +/// Boa test262 tester +#[derive(StructOpt, Debug)] +#[structopt(name = "Boa test262 tester")] +struct Cli { + // Whether to show verbose output. + #[structopt(short, long)] + verbose: bool, + + /// Path to the Test262 suite. + #[structopt(short, long, parse(from_os_str), default_value = "./test262")] + suite: PathBuf, + + // Run the tests only in parsing mode. + #[structopt(short = "p", long)] + only_parse: bool, +} + +/// Program entry point. +fn main() { + let cli = Cli::from_args(); + + let harness = + read_harness(cli.suite.as_path()).expect("could not read initialization bindings"); + + let global_suite = + read_global_suite(cli.suite.as_path()).expect("could not get the list of tests to run"); + + let results = global_suite.run(&harness); + + println!("Results:"); + println!("Total tests: {}", results.total_tests); + println!("Passed tests: {}", results.passed_tests); + println!( + "Conformance: {:.2}%", + (results.passed_tests as f64 / results.total_tests as f64) * 100.0 + ) +} + +/// All the harness include files. +#[derive(Debug, Clone)] +struct Harness { + assert: Box, + sta: Box, + includes: FxHashMap, Box>, +} + +/// Represents a test suite. +#[derive(Debug, Clone)] +struct TestSuite { + name: Box, + suites: Box<[TestSuite]>, + tests: Box<[Test]>, +} + +/// Outcome of a test suite. +#[derive(Debug, Clone)] +struct SuiteResult { + name: Box, + passed: bool, + total_tests: usize, + passed_tests: usize, + ignored_tests: usize, + suites: Box<[SuiteResult]>, + tests: Box<[TestResult]>, +} + +/// Outcome of a test. +#[derive(Debug, Clone)] +struct TestResult { + name: Box, + passed: Option, +} + +/// Represents a test. +#[derive(Debug, Clone)] +struct Test { + name: Box, + description: Box, + esid: Option>, + flags: TestFlags, + information: Box, + features: Box<[Box]>, + expected_outcome: Outcome, + includes: Box<[Box]>, + locale: Locale, + content: Box, +} + +impl Test { + /// Creates a new test. + #[inline] + fn new(name: N, content: C, metadata: MetaData) -> Self + where + N: Into>, + C: Into>, + { + Self { + name: name.into(), + description: metadata.description, + esid: metadata.esid, + flags: metadata.flags.into(), + information: metadata.info, + features: metadata.features, + expected_outcome: Outcome::from(metadata.negative), + includes: metadata.includes, + locale: metadata.locale, + content: content.into(), + } + } +} + +/// An outcome for a test. +#[derive(Debug, Clone)] +enum Outcome { + Positive, + Negative { phase: Phase, error_type: Box }, +} + +impl Default for Outcome { + fn default() -> Self { + Self::Positive + } +} + +impl From> for Outcome { + fn from(neg: Option) -> Self { + neg.map(|neg| Self::Negative { + phase: neg.phase, + error_type: neg.error_type, + }) + .unwrap_or_default() + } +} + +bitflags! { + struct TestFlags: u16 { + const STRICT = 0b000000001; + const NO_STRICT = 0b000000010; + const MODULE = 0b000000100; + const RAW = 0b000001000; + const ASYNC = 0b000010000; + const GENERATED = 0b000100000; + const CAN_BLOCK_IS_FALSE = 0b001000000; + const CAN_BLOCK_IS_TRUE = 0b010000000; + const NON_DETERMINISTIC = 0b100000000; + } +} + +impl Default for TestFlags { + fn default() -> Self { + Self::STRICT | Self::NO_STRICT + } +} + +impl From for TestFlags { + fn from(flag: TestFlag) -> Self { + match flag { + TestFlag::OnlyStrict => Self::STRICT, + TestFlag::NoStrict => Self::NO_STRICT, + TestFlag::Module => Self::MODULE, + TestFlag::Raw => Self::RAW, + TestFlag::Async => Self::ASYNC, + TestFlag::Generated => Self::GENERATED, + TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, + TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, + TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, + } + } +} + +impl From for TestFlags +where + T: AsRef<[TestFlag]>, +{ + fn from(flags: T) -> Self { + let flags = flags.as_ref(); + if flags.is_empty() { + Self::default() + } else { + let mut result = Self::empty(); + for flag in flags { + result |= Self::from(*flag); + } + + if !result.intersects(Self::default()) { + result |= Self::default() + } + + result + } + } +} + +/// Phase for an error. +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum Phase { + Parse, + Early, + Resolution, + Runtime, +} + +/// Locale information structure. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(transparent)] +struct Locale { + locale: Box<[Box]>, +} diff --git a/tester/src/read.rs b/tester/src/read.rs new file mode 100644 index 00000000000..28c2d451a11 --- /dev/null +++ b/tester/src/read.rs @@ -0,0 +1,185 @@ +//! Module to read the list of test suites from disk. + +use super::{Harness, Locale, Phase, Test, TestSuite}; +use fxhash::FxHashMap; +use serde::Deserialize; +use std::{fs, io, path::Path}; + +/// Representation of the YAML metadata in Test262 tests. +#[derive(Debug, Clone, Deserialize)] +pub(super) struct MetaData { + pub(super) description: Box, + pub(super) esid: Option>, + pub(super) es5id: Option>, + pub(super) es6id: Option>, + #[serde(default)] + pub(super) info: Box, + #[serde(default)] + pub(super) features: Box<[Box]>, + #[serde(default)] + pub(super) includes: Box<[Box]>, + #[serde(default)] + pub(super) flags: Box<[TestFlag]>, + #[serde(default)] + pub(super) negative: Option, + #[serde(default)] + pub(super) locale: Locale, +} + +/// Negative test information structure. +#[derive(Debug, Clone, Deserialize)] +pub(super) struct Negative { + pub(super) phase: Phase, + #[serde(rename = "type")] + pub(super) error_type: Box, +} + +/// Individual test flag. +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum TestFlag { + OnlyStrict, + NoStrict, + Module, + Raw, + Async, + Generated, + #[serde(rename = "CanBlockIsFalse")] + CanBlockIsFalse, + #[serde(rename = "CanBlockIsTrue")] + CanBlockIsTrue, + #[serde(rename = "non-deterministic")] + NonDeterministic, +} + +/// Reads the Test262 defined bindings. +pub(super) fn read_harness(test262: &Path) -> io::Result { + let mut includes = FxHashMap::default(); + + for entry in fs::read_dir(test262.join("harness"))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + + if file_name == "assert.js" || file_name == "sta.js" { + continue; + } + + let content = fs::read_to_string(entry.path())?; + + includes.insert( + file_name.into_owned().into_boxed_str(), + content.into_boxed_str(), + ); + } + let assert = fs::read_to_string(test262.join("harness/assert.js"))?.into_boxed_str(); + let sta = fs::read_to_string(test262.join("harness/sta.js"))?.into_boxed_str(); + + Ok(Harness { + assert, + sta, + includes, + }) +} + +/// Reads the global suite from disk. +pub(super) fn read_global_suite(test262: &Path) -> io::Result { + let path = test262.join("test"); + + read_suite(path.as_path()) +} + +/// Reads a test suite in the given path. +fn read_suite(path: &Path) -> io::Result { + use std::ffi::OsStr; + + let name = path + .file_stem() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("test suite with no name found: {}", path.display()), + ) + })? + .to_str() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("non-UTF-8 suite name found: {}", path.display()), + ) + })?; + + let mut suites = Vec::new(); + let mut tests = Vec::new(); + + let filter = |st: &OsStr| { + st.to_string_lossy().ends_with("_FIXTURE.js") + // TODO: see if we can fix this. + || st.to_string_lossy() == "line-terminator-normalisation-CR.js" + }; + + // TODO: iterate in parallel + for entry in path.read_dir()? { + let entry = entry?; + + if entry.file_type()?.is_dir() { + suites.push(read_suite(entry.path().as_path())?); + } else if filter(&entry.file_name()) { + continue; + } else { + tests.push(read_test(entry.path().as_path())?); + } + } + + Ok(TestSuite { + name: name.into(), + suites: suites.into_boxed_slice(), + tests: tests.into_boxed_slice(), + }) +} + +/// Reads information about a given test case. +fn read_test(path: &Path) -> io::Result { + let name = path + .file_stem() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("test with no file name found: {}", path.display()), + ) + })? + .to_str() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("non-UTF-8 file name found: {}", path.display()), + ) + })?; + + let content = fs::read_to_string(path)?; + + let metadata = read_metadata(&content)?; + + Ok(Test::new(name, content, metadata)) +} + +/// Reads the metadata from the input test code. +fn read_metadata(code: &str) -> io::Result { + use once_cell::sync::Lazy; + use regex::Regex; + + /// Regular expression to retrieve the metadata of a test. + static META_REGEX: Lazy = Lazy::new(|| { + Regex::new(r#"/\*\-{3}((?:.|\n)*)\-{3}\*/"#) + .expect("could not compile metadata regular expression") + }); + + let yaml = META_REGEX + .captures(code) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "no metadata found"))? + .get(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "no metadata found"))? + .as_str(); + + serde_yaml::from_str(yaml).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +}