Skip to content

Commit

Permalink
Seperate boa_tester SuiteResults stats by es version.
Browse files Browse the repository at this point in the history 
this allows us to organize total, passed, ignored, and panic stats by es version
  • Loading branch information
@ZackMitkin
ZackMitkin committed Mar 20, 2023
1 parent 6f43cb7 commit 077e136
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 108 deletions.
107 changes: 48 additions & 59 deletions boa_tester/src/exec/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
mod js262;

use crate::{
read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult,
TestResult, TestSuite,
read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test,
TestFlags, TestOutcomeResult, TestResult, TestSuite,
};
use boa_engine::{
context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction,
Expand Down Expand Up @@ -58,80 +58,75 @@ impl TestSuite {
}

// Count passed tests and es specs
let mut passed = 0;
let mut ignored = 0;
let mut panic = 0;
let mut es5_passed = 0;
let mut es6_passed = 0;
let mut es5_total = 0;
let mut es6_total = 0;
let mut all = Statistics::default();
let mut es5 = Statistics::default();
let mut es6 = Statistics::default();

let mut append_stats = |spec_version: &SpecVersion, f: &dyn Fn(&mut Statistics)| {
f(&mut all);
if spec_version == &SpecVersion::ES5 {
f(&mut es5);
} else if spec_version == &SpecVersion::ES6 {
f(&mut es6);
}
};

for test in &tests {
match test.result {
TestOutcomeResult::Passed => {
passed += 1;
if test.es5 {
es5_passed += 1;
}
if test.es6 {
es6_passed += 1;
}
append_stats(&test.spec_version, &|stats| {
stats.passed += 1;
});
}
TestOutcomeResult::Ignored => {
append_stats(&test.spec_version, &|stats| {
stats.ignored += 1;
});
}
TestOutcomeResult::Panic => {
append_stats(&test.spec_version, &|stats| {
stats.panic += 1;
});
}
TestOutcomeResult::Ignored => ignored += 1,
TestOutcomeResult::Panic => panic += 1,
TestOutcomeResult::Failed => {}
}
if test.es5 {
es5_total += 1;
}
if test.es6 {
es6_total += 1;
}
append_stats(&test.spec_version, &|stats| {
stats.total += 1;
});
}

// Count total tests
let mut total = tests.len();
for suite in &suites {
total += suite.total;
passed += suite.passed;
ignored += suite.ignored;
panic += suite.panic;
es5_total += suite.es5_total;
es6_total += suite.es6_total;
es5_passed += suite.es5_passed;
es6_passed += suite.es6_passed;
all = all + suite.all_stats.clone();
es5 = es5 + suite.es5_stats.clone();
es6 = es6 + suite.es6_stats.clone();
features.append(&mut suite.features.clone());
}

if verbose != 0 {
println!(
"Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \
"Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \
{}{}), conformance: {:.2}%",
all.total,
self.path.display(),
passed.to_string().green(),
ignored.to_string().yellow(),
(total - passed - ignored).to_string().red(),
if panic == 0 {
all.passed.to_string().green(),
all.ignored.to_string().yellow(),
(all.total - all.passed - all.ignored).to_string().red(),
if all.panic == 0 {
"0".normal()
} else {
panic.to_string().red()
all.panic.to_string().red()
},
if panic == 0 { "" } else { " ⚠" }.red(),
(passed as f64 / total as f64) * 100.0
if all.panic == 0 { "" } else { " ⚠" }.red(),
(all.passed as f64 / all.total as f64) * 100.0
);
}

SuiteResult {
name: self.name.clone(),
total,
passed,
ignored,
panic,
all_stats: all,
es5_stats: es5,
es6_stats: es6,
suites,
es5_total,
es6_total,
es5_passed,
es6_passed,
tests,
features,
}
Expand Down Expand Up @@ -168,9 +163,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es5id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file.")
Expand All @@ -189,9 +182,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es6id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
Expand Down Expand Up @@ -390,9 +381,7 @@ impl Test {

TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es6id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result,
result_text: result_text.into_boxed_str(),
Expand Down
94 changes: 57 additions & 37 deletions boa_tester/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ use serde::{
use std::{
fs::{self, File},
io::Read,
ops::Add,
path::{Path, PathBuf},
};

Expand Down Expand Up @@ -270,29 +271,32 @@ fn run_test_suite(
}
let results = suite.run(&harness, verbose, parallel);

let total = results.all_stats.total;
let passed = results.all_stats.passed;
let ignored = results.all_stats.ignored;
let panicked = results.all_stats.panic;

println!();
println!("Results:");
println!("Total tests: {}", results.total);
println!("Passed tests: {}", results.passed.to_string().green());
println!("Ignored tests: {}", results.ignored.to_string().yellow());
println!("Total tests: {total}");
println!("Passed tests: {}", passed.to_string().green());
println!("Ignored tests: {}", ignored.to_string().yellow());
println!(
"Failed tests: {} (panics: {})",
(results.total - results.passed - results.ignored)
.to_string()
.red(),
results.panic.to_string().red()
(total - passed - ignored).to_string().red(),
panicked.to_string().red()
);
println!(
"Conformance: {:.2}%",
(results.passed as f64 / results.total as f64) * 100.0
(passed as f64 / total as f64) * 100.0
);
println!(
"ES5 Conformance: {:.2}%",
(results.es5_passed as f64 / results.es5_total as f64) * 100.0
(results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0
);
println!(
"ES6 Conformance: {:.2}%",
(results.es6_passed as f64 / results.es6_total as f64) * 100.0
(results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0
);

write_json(results, output, verbose)
Expand Down Expand Up @@ -326,11 +330,9 @@ struct TestSuite {
tests: Box<[Test]>,
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
/// Represents a tests statistic
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
struct Statistics {
#[serde(rename = "c")]
total: usize,
#[serde(rename = "o")]
Expand All @@ -339,14 +341,32 @@ struct SuiteResult {
ignored: usize,
#[serde(rename = "p")]
panic: usize,
#[serde(rename = "5t", default)]
es5_total: usize,
#[serde(rename = "6t", default)]
es6_total: usize,
#[serde(rename = "5p", default)]
es5_passed: usize,
#[serde(rename = "6p", default)]
es6_passed: usize,
}

impl Add for Statistics {
type Output = Self;

fn add(self, rhs: Self) -> Self::Output {
Self {
total: self.total + rhs.total,
passed: self.passed + rhs.passed,
ignored: self.ignored + rhs.ignored,
panic: self.panic + rhs.panic,
}
}
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
all_stats: Statistics,
#[serde(rename = "a5")]
es5_stats: Statistics,
#[serde(rename = "a6")]
es6_stats: Statistics,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(rename = "s")]
suites: Vec<SuiteResult>,
Expand All @@ -364,12 +384,8 @@ struct SuiteResult {
struct TestResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "v", default)]
spec_version: Option<i8>,
#[serde(default)]
es5: bool,
#[serde(default)]
es6: bool,
#[serde(rename = "v")]
spec_version: SpecVersion,
#[serde(rename = "s", default)]
strict: bool,
#[serde(skip)]
Expand All @@ -390,16 +406,22 @@ enum TestOutcomeResult {
Panic,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
enum SpecVersion {
ES5 = 5,
ES6 = 6,
ES13 = 13,
}

/// Represents a test.
#[derive(Debug, Clone)]
#[allow(dead_code)]
struct Test {
name: Box<str>,
description: Box<str>,
esid: Option<Box<str>>,
spec_version: Option<i8>,
es5id: Option<Box<str>>,
es6id: Option<Box<str>>,
spec_version: SpecVersion,
flags: TestFlags,
information: Box<str>,
features: Box<[Box<str>]>,
Expand All @@ -418,20 +440,18 @@ impl Test {
C: Into<Box<Path>>,
{
let spec_version = if metadata.es5id.is_some() {
Some(5)
SpecVersion::ES5
} else if metadata.es6id.is_some() {
Some(6)
SpecVersion::ES6
} else {
None
SpecVersion::ES13
};

Self {
name: name.into(),
description: metadata.description,
esid: metadata.esid,
spec_version,
es5id: metadata.es5id,
es6id: metadata.es6id,
flags: metadata.flags.into(),
information: metadata.info,
features: metadata.features,
Expand Down
24 changes: 12 additions & 12 deletions boa_tester/src/results.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,10 @@ impl From<ResultInfo> for ReducedResultInfo {
Self {
commit: info.commit,
test262_commit: info.test262_commit,
total: info.results.total,
passed: info.results.passed,
ignored: info.results.ignored,
panic: info.results.panic,
total: info.results.all_stats.total,
passed: info.results.all_stats.passed,
ignored: info.results.all_stats.ignored,
panic: info.results.all_stats.panic,
}
}
}
Expand Down Expand Up @@ -220,24 +220,24 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result
))
.wrap_err("could not read the new results")?;

let base_total = base_results.results.total as isize;
let new_total = new_results.results.total as isize;
let base_total = base_results.results.all_stats.total as isize;
let new_total = new_results.results.all_stats.total as isize;
let total_diff = new_total - base_total;

let base_passed = base_results.results.passed as isize;
let new_passed = new_results.results.passed as isize;
let base_passed = base_results.results.all_stats.passed as isize;
let new_passed = new_results.results.all_stats.passed as isize;
let passed_diff = new_passed - base_passed;

let base_ignored = base_results.results.ignored as isize;
let new_ignored = new_results.results.ignored as isize;
let base_ignored = base_results.results.all_stats.ignored as isize;
let new_ignored = new_results.results.all_stats.ignored as isize;
let ignored_diff = new_ignored - base_ignored;

let base_failed = base_total - base_passed - base_ignored;
let new_failed = new_total - new_passed - new_ignored;
let failed_diff = new_failed - base_failed;

let base_panics = base_results.results.panic as isize;
let new_panics = new_results.results.panic as isize;
let base_panics = base_results.results.all_stats.panic as isize;
let new_panics = new_results.results.all_stats.panic as isize;
let panic_diff = new_panics - base_panics;

let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;
Expand Down

0 comments on commit 077e136

Please sign in to comment.