From 7cb623b33d149e0f1de8cdf2ebaa2bab19cf5be1 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 14 Mar 2024 14:16:57 -0700 Subject: [PATCH 1/4] Allow a "skippable" runner. This runner will inspect Ok values to determine if the test should be ignored ("skipped" for our purposes means runtime ignore). This can be seem in terse or json output directly. In order to keep default pretty printing identical to libtest, the actual conclusion line is unmodified. --- src/lib.rs | 29 +++++++++++++++++++++++++++++ src/printer.rs | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 1cb6f29..d7e27a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -122,6 +122,31 @@ impl Trial { } } + /// Creates a (non-benchmark) test with the given name and runner. + /// + /// The runner returning `Ok(None)` is interpreted as the test passing. The runner returning + /// `Ok(Some())` is interpreted as the test being skipped for reasons determined at + /// runtime. The reason may be passed as a [`String`] if desired. If the runner returns + /// `Err(_)`, the test is considered failed. + pub fn skippable_test(name: impl Into, runner: R) -> Self + where + R: FnOnce() -> Result, Failed> + Send + 'static, + { + Self { + runner: Box::new(|_test_mode| match runner() { + Ok(None) => Outcome::Passed, + Ok(Some(_reason)) => Outcome::RuntimeIgnored, + Err(failed) => Outcome::Failed(failed), + }), + info: TestInfo { + name: name.into(), + kind: String::new(), + is_ignored: false, + is_bench: false, + }, + } + } + /// Creates a benchmark with the given name and runner. /// /// If the runner's parameter `test_mode` is `true`, the runner function @@ -303,6 +328,9 @@ enum Outcome { /// The test or benchmark was ignored. Ignored, + /// The test or benchmark was ignored. + RuntimeIgnored, + /// The benchmark was successfully run. Measured(Measurement), } @@ -476,6 +504,7 @@ pub fn run(args: &Arguments, mut tests: Vec) -> Conclusion { }, Outcome::Ignored => conclusion.num_ignored += 1, Outcome::Measured(_) => conclusion.num_measured += 1, + Outcome::RuntimeIgnored => conclusion.num_ignored += 1, } }; diff --git a/src/printer.rs b/src/printer.rs index c3dde85..f13c86d 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -158,6 +158,7 @@ impl Printer { writeln!(self.out).unwrap(); return; } + Outcome::RuntimeIgnored => 'S', }; self.out.set_color(&color_of_outcome(outcome)).unwrap(); @@ -184,6 +185,7 @@ impl Printer { Outcome::Failed(_) => "failed", Outcome::Ignored => "ignored", Outcome::Measured(_) => unreachable!(), + Outcome::RuntimeIgnored => "skipped", }, match outcome { Outcome::Failed(Failed { msg: Some(msg) }) => { @@ -317,6 +319,7 @@ impl Printer { Outcome::Failed { .. } => "FAILED", Outcome::Ignored => "ignored", Outcome::Measured { .. } => "bench", + Outcome::RuntimeIgnored => "skipped", }; self.out.set_color(&color_of_outcome(outcome)).unwrap(); @@ -354,6 +357,7 @@ fn color_of_outcome(outcome: &Outcome) -> ColorSpec { Outcome::Failed { .. } => Color::Red, Outcome::Ignored => Color::Yellow, Outcome::Measured { .. } => Color::Cyan, + Outcome::RuntimeIgnored => Color::Blue, }; out.set_fg(Some(color)); out From a47a4d55add1d1df6e4151961008ceefda2e88a4 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 14 Mar 2024 14:51:46 -0700 Subject: [PATCH 2/4] tests: Use reordered logging This change will help the ability to check if things run out of order. --- tests/all_passing.rs | 9 ++------- tests/common/mod.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/all_passing.rs b/tests/all_passing.rs index b5c5552..40be8d2 100644 --- a/tests/all_passing.rs +++ b/tests/all_passing.rs @@ -2,7 +2,7 @@ use common::{args, check}; use libtest_mimic::{Trial, Conclusion}; use pretty_assertions::assert_eq; -use crate::common::do_run; +use crate::common::{assert_reordered_log, conclusion_to_output, do_run}; #[macro_use] mod common; @@ -153,10 +153,5 @@ fn terse_output() { num_ignored: 0, num_measured: 0, }); - assert_log!(out, " - running 3 tests - ... - test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; \ - finished in 0.00s - "); + assert_reordered_log(out.as_str(), 3, &["..."], &conclusion_to_output(&c)); } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c2c16a2..199fa85 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -135,8 +135,14 @@ pub fn check( assert_eq!(c, expected_conclusion); } -fn conclusion_to_output(c: &Conclusion) -> String { - let Conclusion { num_filtered_out, num_passed, num_failed, num_ignored, num_measured } = *c; +pub fn conclusion_to_output(c: &Conclusion) -> String { + let Conclusion { + num_filtered_out, + num_passed, + num_failed, + num_ignored, + num_measured, + } = *c; format!( "test result: {}. {} passed; {} failed; {} ignored; {} measured; {} filtered out;", if num_failed > 0 { "FAILED" } else { "ok" }, From 905454918e2734b83bb3dbf62e011573bcd6f75f Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 14 Mar 2024 15:06:14 -0700 Subject: [PATCH 3/4] Allow sloopy sorting Because tests may run out of order, it is sometimes useful to have lines checked without specific ordering. --- tests/all_passing.rs | 2 +- tests/common/mod.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/all_passing.rs b/tests/all_passing.rs index 40be8d2..32062b3 100644 --- a/tests/all_passing.rs +++ b/tests/all_passing.rs @@ -153,5 +153,5 @@ fn terse_output() { num_ignored: 0, num_measured: 0, }); - assert_reordered_log(out.as_str(), 3, &["..."], &conclusion_to_output(&c)); + assert_reordered_log(out.as_str(), 3, &["..."], &conclusion_to_output(&c), false); } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 199fa85..0513bc0 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -52,7 +52,7 @@ pub fn clean_expected_log(s: &str) -> String { /// Best effort tool to check certain things about a log that might have all /// tests randomly ordered. -pub fn assert_reordered_log(actual: &str, num: u64, expected_lines: &[&str], tail: &str) { +pub fn assert_reordered_log(actual: &str, num: u64, expected_lines: &[&str], tail: &str, sloppy: bool) { let actual = actual.trim(); let (first_line, rest) = actual.split_once('\n').expect("log has too few lines"); let (middle, last_line) = rest.rsplit_once('\n').expect("log has too few lines"); @@ -63,6 +63,15 @@ pub fn assert_reordered_log(actual: &str, num: u64, expected_lines: &[&str], tai let mut actual_lines = HashMap::new(); for line in middle.lines().map(|l| l.trim()).filter(|l| !l.is_empty()) { + // If we're testing sloppily, sort before adding the line so we can compare sorted strings. + // This is effectively testing if the strings are anagrams/permutations. + let line = if sloppy { + let mut line_str = line.chars().collect::>(); + line_str.sort(); + line_str.iter().collect::() + } else { + line.to_string() + }; *actual_lines.entry(line).or_insert(0) += 1; } @@ -131,6 +140,7 @@ pub fn check( num_running_tests, &expected_output.lines().collect::>(), &conclusion_to_output(&c), + false, ); assert_eq!(c, expected_conclusion); } From 3f142cdf48eeb0d8b826dd3c89f04a531f003048 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 14 Mar 2024 21:29:05 -0700 Subject: [PATCH 4/4] Add tests for skipped tests --- tests/all_passing.rs | 51 +++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/tests/all_passing.rs b/tests/all_passing.rs index 32062b3..c4331f3 100644 --- a/tests/all_passing.rs +++ b/tests/all_passing.rs @@ -13,23 +13,25 @@ fn tests() -> Vec { Trial::test("foo", || Ok(())), Trial::test("bar", || Ok(())), Trial::test("barro", || Ok(())), + Trial::skippable_test("baz", || Ok(Some("Can't find a quux".into()))), ] } #[test] fn normal() { - check(args([]), tests, 3, + check(args([]), tests, 4, Conclusion { num_filtered_out: 0, num_passed: 3, num_failed: 0, - num_ignored: 0, + num_ignored: 1, num_measured: 0, }, " test foo ... ok test bar ... ok test barro ... ok + test baz ... skipped " ); } @@ -38,7 +40,7 @@ fn normal() { fn filter_one() { check(args(["foo"]), tests, 1, Conclusion { - num_filtered_out: 2, + num_filtered_out: 3, num_passed: 1, num_failed: 0, num_ignored: 0, @@ -52,7 +54,7 @@ fn filter_one() { fn filter_two() { check(args(["bar"]), tests, 2, Conclusion { - num_filtered_out: 1, + num_filtered_out: 2, num_passed: 2, num_failed: 0, num_ignored: 0, @@ -70,7 +72,7 @@ fn filter_two() { fn filter_exact() { check(args(["bar", "--exact"]), tests, 1, Conclusion { - num_filtered_out: 2, + num_filtered_out: 3, num_passed: 1, num_failed: 0, num_ignored: 0, @@ -84,7 +86,7 @@ fn filter_exact() { fn filter_two_and_skip() { check(args(["--skip", "barro", "bar"]), tests, 1, Conclusion { - num_filtered_out: 2, + num_filtered_out: 3, num_passed: 1, num_failed: 0, num_ignored: 0, @@ -94,51 +96,70 @@ fn filter_two_and_skip() { ); } +#[test] +fn filter_runtime_ignored() { + check(args(["baz", "--exact"]), tests, 1, + Conclusion { + num_filtered_out: 3, + num_passed: 0, + num_failed: 0, + num_ignored: 1, + num_measured: 0, + }, + "test baz ... skipped", + ); +} + #[test] fn skip_nothing() { - check(args(["--skip", "peter"]), tests, 3, + check(args(["--skip", "peter"]), tests, 4, Conclusion { num_filtered_out: 0, num_passed: 3, num_failed: 0, - num_ignored: 0, + num_ignored: 1, num_measured: 0, }, " test foo ... ok test bar ... ok test barro ... ok + test baz ... skipped " ); } #[test] fn skip_two() { - check(args(["--skip", "bar"]), tests, 1, + check(args(["--skip", "bar"]), tests, 2, Conclusion { num_filtered_out: 2, num_passed: 1, num_failed: 0, - num_ignored: 0, + num_ignored: 1, num_measured: 0, }, - "test foo ... ok" + " + test foo ... ok + test baz ... skipped + " ); } #[test] fn skip_exact() { - check(args(["--exact", "--skip", "bar"]), tests, 2, + check(args(["--exact", "--skip", "bar"]), tests, 3, Conclusion { num_filtered_out: 1, num_passed: 2, num_failed: 0, - num_ignored: 0, + num_ignored: 1, num_measured: 0, }, " test foo ... ok test barro ... ok + test baz ... skipped " ); } @@ -150,8 +171,8 @@ fn terse_output() { num_filtered_out: 0, num_passed: 3, num_failed: 0, - num_ignored: 0, + num_ignored: 1, num_measured: 0, }); - assert_reordered_log(out.as_str(), 3, &["..."], &conclusion_to_output(&c), false); + assert_reordered_log(out.as_str(), 4, &["...S"], &conclusion_to_output(&c), true); }