Skip to content

Commit

Permalink
Add ES5 and ES6 Conformance calculation to boa_tester (#2690)
Browse files Browse the repository at this point in the history 
This is calculated based the tests `es5id` or `es6id`.  

Judging by [this](tc39/test262#1557) it's probably not the best method of doing it, but the alternative would require a lot of rework on the boa_tester so that it can pull an older version of the test262 spec which has it's own problems since there's not really an "ES5" only version.
 
I would think that if we're 100% passing on es5id's then it's safe to assume boa supports es5 (assuming test262's es5id is covering all the es5 cases)

This Pull Request fixes/closes #2629.

It changes the following:
- Store `spec_version` in TestResult based on the tests `es[6/5]id`
- Count all es5, es6 and their test outcome during `TestSuite::run()`
- Print the conformance.

I'm serializing the `spec_version` outcomes so that it can be displayed in test262 github page. I'd like to work on that too if possible. Let me know if there's anything more I should cover in this PR, I'll gladly do it :)

Co-authored-by: jedel1043 <[email protected]>
  • Loading branch information
@ZackMitkin @jedel1043
ZackMitkin and jedel1043 committed Mar 20, 2023
1 parent 431a358 commit 02f737a
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 61 deletions.
79 changes: 52 additions & 27 deletions boa_tester/src/exec/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
mod js262;

use crate::{
read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult,
TestResult, TestSuite,
read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test,
TestFlags, TestOutcomeResult, TestResult, TestSuite,
};
use boa_engine::{
context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction,
Expand Down Expand Up @@ -57,53 +57,75 @@ impl TestSuite {
println!();
}

// Count passed tests
let mut passed = 0;
let mut ignored = 0;
let mut panic = 0;
// Count passed tests and es specs
let mut all = Statistics::default();
let mut es5 = Statistics::default();
let mut es6 = Statistics::default();

let mut append_stats = |spec_version: SpecVersion, f: &dyn Fn(&mut Statistics)| {
f(&mut all);
if spec_version == SpecVersion::ES5 {
f(&mut es5);
} else if spec_version == SpecVersion::ES6 {
f(&mut es6);
}
};

for test in &tests {
match test.result {
TestOutcomeResult::Passed => passed += 1,
TestOutcomeResult::Ignored => ignored += 1,
TestOutcomeResult::Panic => panic += 1,
TestOutcomeResult::Passed => {
append_stats(test.spec_version, &|stats| {
stats.passed += 1;
});
}
TestOutcomeResult::Ignored => {
append_stats(test.spec_version, &|stats| {
stats.ignored += 1;
});
}
TestOutcomeResult::Panic => {
append_stats(test.spec_version, &|stats| {
stats.panic += 1;
});
}
TestOutcomeResult::Failed => {}
}
append_stats(test.spec_version, &|stats| {
stats.total += 1;
});
}

// Count total tests
let mut total = tests.len();
for suite in &suites {
total += suite.total;
passed += suite.passed;
ignored += suite.ignored;
panic += suite.panic;
all = all + suite.all_stats.clone();
es5 = es5 + suite.es5_stats.clone();
es6 = es6 + suite.es6_stats.clone();
features.append(&mut suite.features.clone());
}

if verbose != 0 {
println!(
"Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \
"Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \
{}{}), conformance: {:.2}%",
all.total,
self.path.display(),
passed.to_string().green(),
ignored.to_string().yellow(),
(total - passed - ignored).to_string().red(),
if panic == 0 {
all.passed.to_string().green(),
all.ignored.to_string().yellow(),
(all.total - all.passed - all.ignored).to_string().red(),
if all.panic == 0 {
"0".normal()
} else {
panic.to_string().red()
all.panic.to_string().red()
},
if panic == 0 { "" } else { " ⚠" }.red(),
(passed as f64 / total as f64) * 100.0
if all.panic == 0 { "" } else { " ⚠" }.red(),
(all.passed as f64 / all.total as f64) * 100.0
);
}

SuiteResult {
name: self.name.clone(),
total,
passed,
ignored,
panic,
all_stats: all,
es5_stats: es5,
es6_stats: es6,
suites,
tests,
features,
Expand Down Expand Up @@ -141,6 +163,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
spec_version: self.spec_version,
strict,
result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file.")
Expand All @@ -159,6 +182,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
spec_version: self.spec_version,
strict,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
Expand Down Expand Up @@ -357,6 +381,7 @@ impl Test {

TestResult {
name: self.name.clone(),
spec_version: self.spec_version,
strict,
result,
result_text: result_text.into_boxed_str(),
Expand Down
85 changes: 71 additions & 14 deletions boa_tester/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ use serde::{
use std::{
fs::{self, File},
io::Read,
ops::Add,
path::{Path, PathBuf},
};

Expand Down Expand Up @@ -270,21 +271,32 @@ fn run_test_suite(
}
let results = suite.run(&harness, verbose, parallel);

let total = results.all_stats.total;
let passed = results.all_stats.passed;
let ignored = results.all_stats.ignored;
let panicked = results.all_stats.panic;

println!();
println!("Results:");
println!("Total tests: {}", results.total);
println!("Passed tests: {}", results.passed.to_string().green());
println!("Ignored tests: {}", results.ignored.to_string().yellow());
println!("Total tests: {total}");
println!("Passed tests: {}", passed.to_string().green());
println!("Ignored tests: {}", ignored.to_string().yellow());
println!(
"Failed tests: {} (panics: {})",
(results.total - results.passed - results.ignored)
.to_string()
.red(),
results.panic.to_string().red()
(total - passed - ignored).to_string().red(),
panicked.to_string().red()
);
println!(
"Conformance: {:.2}%",
(results.passed as f64 / results.total as f64) * 100.0
(passed as f64 / total as f64) * 100.0
);
println!(
"ES5 Conformance: {:.2}%",
(results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0
);
println!(
"ES6 Conformance: {:.2}%",
(results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0
);

write_json(results, output, verbose)
Expand Down Expand Up @@ -318,19 +330,43 @@ struct TestSuite {
tests: Box<[Test]>,
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "c")]
/// Represents a tests statistic
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
struct Statistics {
#[serde(rename = "t")]
total: usize,
#[serde(rename = "o")]
passed: usize,
#[serde(rename = "i")]
ignored: usize,
#[serde(rename = "p")]
panic: usize,
}

impl Add for Statistics {
type Output = Self;

fn add(self, rhs: Self) -> Self::Output {
Self {
total: self.total + rhs.total,
passed: self.passed + rhs.passed,
ignored: self.ignored + rhs.ignored,
panic: self.panic + rhs.panic,
}
}
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
all_stats: Statistics,
#[serde(rename = "a5", default)]
es5_stats: Statistics,
#[serde(rename = "a6", default)]
es6_stats: Statistics,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(rename = "s")]
suites: Vec<SuiteResult>,
Expand All @@ -348,6 +384,8 @@ struct SuiteResult {
struct TestResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "v", default)]
spec_version: SpecVersion,
#[serde(rename = "s", default)]
strict: bool,
#[serde(skip)]
Expand All @@ -368,13 +406,23 @@ enum TestOutcomeResult {
Panic,
}

#[derive(Debug, Serialize, Clone, Copy, Deserialize, PartialEq, Default)]
#[serde(untagged)]
enum SpecVersion {
ES5 = 5,
ES6 = 6,
#[default]
ES13 = 13,
}

/// Represents a test.
#[derive(Debug, Clone)]
#[allow(dead_code)]
struct Test {
name: Box<str>,
description: Box<str>,
esid: Option<Box<str>>,
spec_version: SpecVersion,
flags: TestFlags,
information: Box<str>,
features: Box<[Box<str>]>,
Expand All @@ -392,10 +440,19 @@ impl Test {
N: Into<Box<str>>,
C: Into<Box<Path>>,
{
let spec_version = if metadata.es5id.is_some() {
SpecVersion::ES5
} else if metadata.es6id.is_some() {
SpecVersion::ES6
} else {
SpecVersion::ES13
};

Self {
name: name.into(),
description: metadata.description,
esid: metadata.esid,
spec_version,
flags: metadata.flags.into(),
information: metadata.info,
features: metadata.features,
Expand Down
39 changes: 19 additions & 20 deletions boa_tester/src/results.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use crate::Statistics;

use super::SuiteResult;
use color_eyre::{eyre::WrapErr, Result};
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -25,14 +27,12 @@ struct ReducedResultInfo {
commit: Box<str>,
#[serde(rename = "u")]
test262_commit: Box<str>,
#[serde(rename = "t")]
total: usize,
#[serde(rename = "o")]
passed: usize,
#[serde(rename = "i")]
ignored: usize,
#[serde(rename = "p")]
panic: usize,
#[serde(rename = "a")]
all_stats: Statistics,
#[serde(rename = "a5", default)]
es5_stats: Statistics,
#[serde(rename = "a6", default)]
es6_stats: Statistics,
}

impl From<ResultInfo> for ReducedResultInfo {
Expand All @@ -41,10 +41,9 @@ impl From<ResultInfo> for ReducedResultInfo {
Self {
commit: info.commit,
test262_commit: info.test262_commit,
total: info.results.total,
passed: info.results.passed,
ignored: info.results.ignored,
panic: info.results.panic,
all_stats: info.results.all_stats,
es5_stats: info.results.es5_stats,
es6_stats: info.results.es6_stats,
}
}
}
Expand Down Expand Up @@ -220,24 +219,24 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result
))
.wrap_err("could not read the new results")?;

let base_total = base_results.results.total as isize;
let new_total = new_results.results.total as isize;
let base_total = base_results.results.all_stats.total as isize;
let new_total = new_results.results.all_stats.total as isize;
let total_diff = new_total - base_total;

let base_passed = base_results.results.passed as isize;
let new_passed = new_results.results.passed as isize;
let base_passed = base_results.results.all_stats.passed as isize;
let new_passed = new_results.results.all_stats.passed as isize;
let passed_diff = new_passed - base_passed;

let base_ignored = base_results.results.ignored as isize;
let new_ignored = new_results.results.ignored as isize;
let base_ignored = base_results.results.all_stats.ignored as isize;
let new_ignored = new_results.results.all_stats.ignored as isize;
let ignored_diff = new_ignored - base_ignored;

let base_failed = base_total - base_passed - base_ignored;
let new_failed = new_total - new_passed - new_ignored;
let failed_diff = new_failed - base_failed;

let base_panics = base_results.results.panic as isize;
let new_panics = new_results.results.panic as isize;
let base_panics = base_results.results.all_stats.panic as isize;
let new_panics = new_results.results.all_stats.panic as isize;
let panic_diff = new_panics - base_panics;

let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;
Expand Down

0 comments on commit 02f737a

Please sign in to comment.