Skip to content

Commit

Permalink
Seperate boa_tester SuiteResults stats by es version.
Browse files Browse the repository at this point in the history 
this allows us to organize total, passed, ignored, and panic stats by es version
  • Loading branch information
@ZackMitkin
ZackMitkin committed Mar 20, 2023
1 parent 6f43cb7 commit eb82ace
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 117 deletions.
107 changes: 48 additions & 59 deletions boa_tester/src/exec/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
mod js262;

use crate::{
read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult,
TestResult, TestSuite,
read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test,
TestFlags, TestOutcomeResult, TestResult, TestSuite,
};
use boa_engine::{
context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction,
Expand Down Expand Up @@ -58,80 +58,75 @@ impl TestSuite {
}

// Count passed tests and es specs
let mut passed = 0;
let mut ignored = 0;
let mut panic = 0;
let mut es5_passed = 0;
let mut es6_passed = 0;
let mut es5_total = 0;
let mut es6_total = 0;
let mut all = Statistics::default();
let mut es5 = Statistics::default();
let mut es6 = Statistics::default();

let mut append_stats = |spec_version: &SpecVersion, f: &dyn Fn(&mut Statistics)| {
f(&mut all);
if spec_version == &SpecVersion::ES5 {
f(&mut es5);
} else if spec_version == &SpecVersion::ES6 {
f(&mut es6);
}
};

for test in &tests {
match test.result {
TestOutcomeResult::Passed => {
passed += 1;
if test.es5 {
es5_passed += 1;
}
if test.es6 {
es6_passed += 1;
}
append_stats(&test.spec_version, &|stats| {
stats.passed += 1;
});
}
TestOutcomeResult::Ignored => {
append_stats(&test.spec_version, &|stats| {
stats.ignored += 1;
});
}
TestOutcomeResult::Panic => {
append_stats(&test.spec_version, &|stats| {
stats.panic += 1;
});
}
TestOutcomeResult::Ignored => ignored += 1,
TestOutcomeResult::Panic => panic += 1,
TestOutcomeResult::Failed => {}
}
if test.es5 {
es5_total += 1;
}
if test.es6 {
es6_total += 1;
}
append_stats(&test.spec_version, &|stats| {
stats.total += 1;
});
}

// Count total tests
let mut total = tests.len();
for suite in &suites {
total += suite.total;
passed += suite.passed;
ignored += suite.ignored;
panic += suite.panic;
es5_total += suite.es5_total;
es6_total += suite.es6_total;
es5_passed += suite.es5_passed;
es6_passed += suite.es6_passed;
all = all + suite.all_stats.clone();
es5 = es5 + suite.es5_stats.clone();
es6 = es6 + suite.es6_stats.clone();
features.append(&mut suite.features.clone());
}

if verbose != 0 {
println!(
"Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \
"Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \
{}{}), conformance: {:.2}%",
all.total,
self.path.display(),
passed.to_string().green(),
ignored.to_string().yellow(),
(total - passed - ignored).to_string().red(),
if panic == 0 {
all.passed.to_string().green(),
all.ignored.to_string().yellow(),
(all.total - all.passed - all.ignored).to_string().red(),
if all.panic == 0 {
"0".normal()
} else {
panic.to_string().red()
all.panic.to_string().red()
},
if panic == 0 { "" } else { " ⚠" }.red(),
(passed as f64 / total as f64) * 100.0
if all.panic == 0 { "" } else { " ⚠" }.red(),
(all.passed as f64 / all.total as f64) * 100.0
);
}

SuiteResult {
name: self.name.clone(),
total,
passed,
ignored,
panic,
all_stats: all,
es5_stats: es5,
es6_stats: es6,
suites,
es5_total,
es6_total,
es5_passed,
es6_passed,
tests,
features,
}
Expand Down Expand Up @@ -168,9 +163,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es5id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file.")
Expand All @@ -189,9 +182,7 @@ impl Test {
}
return TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es6id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
Expand Down Expand Up @@ -390,9 +381,7 @@ impl Test {

TestResult {
name: self.name.clone(),
es5: self.es5id.is_some(),
es6: self.es6id.is_some(),
spec_version: self.spec_version,
spec_version: self.spec_version.clone(),
strict,
result,
result_text: result_text.into_boxed_str(),
Expand Down
96 changes: 58 additions & 38 deletions boa_tester/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ use serde::{
use std::{
fs::{self, File},
io::Read,
ops::Add,
path::{Path, PathBuf},
};

Expand Down Expand Up @@ -270,29 +271,32 @@ fn run_test_suite(
}
let results = suite.run(&harness, verbose, parallel);

let total = results.all_stats.total;
let passed = results.all_stats.passed;
let ignored = results.all_stats.ignored;
let panicked = results.all_stats.panic;

println!();
println!("Results:");
println!("Total tests: {}", results.total);
println!("Passed tests: {}", results.passed.to_string().green());
println!("Ignored tests: {}", results.ignored.to_string().yellow());
println!("Total tests: {total}");
println!("Passed tests: {}", passed.to_string().green());
println!("Ignored tests: {}", ignored.to_string().yellow());
println!(
"Failed tests: {} (panics: {})",
(results.total - results.passed - results.ignored)
.to_string()
.red(),
results.panic.to_string().red()
(total - passed - ignored).to_string().red(),
panicked.to_string().red()
);
println!(
"Conformance: {:.2}%",
(results.passed as f64 / results.total as f64) * 100.0
(passed as f64 / total as f64) * 100.0
);
println!(
"ES5 Conformance: {:.2}%",
(results.es5_passed as f64 / results.es5_total as f64) * 100.0
(results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0
);
println!(
"ES6 Conformance: {:.2}%",
(results.es6_passed as f64 / results.es6_total as f64) * 100.0
(results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0
);

write_json(results, output, verbose)
Expand Down Expand Up @@ -326,27 +330,43 @@ struct TestSuite {
tests: Box<[Test]>,
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "c")]
/// Represents a tests statistic
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
struct Statistics {
#[serde(rename = "t")]
total: usize,
#[serde(rename = "o")]
passed: usize,
#[serde(rename = "i")]
ignored: usize,
#[serde(rename = "p")]
panic: usize,
#[serde(rename = "5t", default)]
es5_total: usize,
#[serde(rename = "6t", default)]
es6_total: usize,
#[serde(rename = "5p", default)]
es5_passed: usize,
#[serde(rename = "6p", default)]
es6_passed: usize,
}

impl Add for Statistics {
type Output = Self;

fn add(self, rhs: Self) -> Self::Output {
Self {
total: self.total + rhs.total,
passed: self.passed + rhs.passed,
ignored: self.ignored + rhs.ignored,
panic: self.panic + rhs.panic,
}
}
}

/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
all_stats: Statistics,
#[serde(rename = "a5")]
es5_stats: Statistics,
#[serde(rename = "a6")]
es6_stats: Statistics,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(rename = "s")]
suites: Vec<SuiteResult>,
Expand All @@ -364,12 +384,8 @@ struct SuiteResult {
struct TestResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "v", default)]
spec_version: Option<i8>,
#[serde(default)]
es5: bool,
#[serde(default)]
es6: bool,
#[serde(rename = "v")]
spec_version: SpecVersion,
#[serde(rename = "s", default)]
strict: bool,
#[serde(skip)]
Expand All @@ -390,16 +406,22 @@ enum TestOutcomeResult {
Panic,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
enum SpecVersion {
ES5 = 5,
ES6 = 6,
ES13 = 13,
}

/// Represents a test.
#[derive(Debug, Clone)]
#[allow(dead_code)]
struct Test {
name: Box<str>,
description: Box<str>,
esid: Option<Box<str>>,
spec_version: Option<i8>,
es5id: Option<Box<str>>,
es6id: Option<Box<str>>,
spec_version: SpecVersion,
flags: TestFlags,
information: Box<str>,
features: Box<[Box<str>]>,
Expand All @@ -418,20 +440,18 @@ impl Test {
C: Into<Box<Path>>,
{
let spec_version = if metadata.es5id.is_some() {
Some(5)
SpecVersion::ES5
} else if metadata.es6id.is_some() {
Some(6)
SpecVersion::ES6
} else {
None
SpecVersion::ES13
};

Self {
name: name.into(),
description: metadata.description,
esid: metadata.esid,
spec_version,
es5id: metadata.es5id,
es6id: metadata.es6id,
flags: metadata.flags.into(),
information: metadata.info,
features: metadata.features,
Expand Down
Loading

0 comments on commit eb82ace

Please sign in to comment.