Skip to content

Commit

Permalink
Merge pull request #167 from trishume/syntest-reports
Browse files Browse the repository at this point in the history 
Syntest reports
  • Loading branch information
@trishume
trishume authored May 28, 2018
2 parents ab5f77d + 3bbce1c commit 42f3da4
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 52 deletions.
1 change: 1 addition & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ script:
fi
- cargo test
- make assets
- make syntest
- rm -Rf examples
- cargo doc
# default features are required for examples to build - so remove them from sight.
Expand Down
19 changes: 14 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,25 @@ info:
$(info - OTHER TARGETS -------------------------------------------------------)
$(info themes | generate default theme pack)
$(info packs | generate default syntax pack)


$(info syntest | run syntax test summary)


$(SUBMODULES):
git submodule update --init --recursive

assets: packs themes

packs: $(SUBMODULES)
cargo run --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump
cargo run --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump

themes: $(SUBMODULES)
cargo run --example gendata -- themepack testdata assets/default.themedump


syntest: $(SUBMODULES)
@echo Tip: Run make update-known-failures to update the known failures file.
cargo run --release --example syntest -- testdata/Packages testdata/Packages --summary | diff -U 1000000 testdata/known_syntest_failures.txt -
@echo No new failures!

update-known-failures: $(SUBMODULES)
cargo run --release --example syntest -- testdata/Packages testdata/Packages --summary | tee testdata/known_syntest_failures.txt

137 changes: 90 additions & 47 deletions examples/syntest.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ extern crate walkdir;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate getopts;

//extern crate onig;
use syntect::parsing::{SyntaxSet, ParseState, ScopeStack, Scope};
use syntect::highlighting::ScopeSelectors;
Expand All @@ -22,9 +24,11 @@ use std::io::{BufRead, BufReader};
use std::fs::File;
use std::cmp::{min, max};
use std::time::Instant;
use walkdir::{DirEntry, WalkDir};
use std::str::FromStr;

use getopts::Options;
use regex::Regex;
use walkdir::{DirEntry, WalkDir};

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SyntaxTestHeaderError {
Expand All @@ -51,6 +55,13 @@ lazy_static! {
)(.*)$"#).unwrap();
}

#[derive(Clone, Copy)]
struct OutputOptions {
time: bool,
debug: bool,
summary: bool,
}

#[derive(Debug)]
struct AssertionRange<'a> {
begin_char: usize,
Expand Down Expand Up @@ -81,7 +92,7 @@ fn get_line_assertion_details<'a>(testtoken_start: &str, testtoken_end: Option<&
if let Some(captures) = SYNTAX_TEST_ASSERTION_PATTERN.captures(&token_and_rest_of_line[testtoken_start.len()..]) {
let mut sst = captures.get(3).unwrap().as_str(); // get the scope selector text
let mut only_whitespace_after_token_end = true;

if let Some(token) = testtoken_end { // if there is an end token defined in the test file header
if let Some(end_token_pos) = sst.find(token) { // and there is an end token in the line
let (ss, after_token_end) = sst.split_at(end_token_pos); // the scope selector text ends at the end token
Expand Down Expand Up @@ -130,7 +141,7 @@ fn process_assertions(assertion: &AssertionRange, test_against_line_scopes: &Vec
}

/// If `parse_test_lines` is `false` then lines that only contain assertions are not parsed
fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: OutputOptions) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
use syntect::util::debug_print_ops;
let f = File::open(path).unwrap();
let mut reader = BufReader::new(f);
Expand All @@ -153,7 +164,9 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
let syntax_file = captures.name("syntax_file").unwrap().as_str();

// find the relevant syntax definition to parse the file with - case is important!
println!("The test file references syntax definition file: {}", syntax_file);
if !out_opts.summary {
println!("The test file references syntax definition file: {}", syntax_file);
}
let syntax = ss.find_syntax_by_path(syntax_file).ok_or(SyntaxTestHeaderError::SyntaxDefinitionNotFound)?;

// iterate over the lines of the file, testing them
Expand All @@ -177,15 +190,17 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
for failure in result.iter().filter(|r|!r.success) {
let length = failure.column_end - failure.column_begin;
let text: String = previous_non_assertion_line.chars().skip(failure.column_begin).take(length).collect();
println!(" Assertion selector {:?} \
from line {:?} failed against line {:?}, column range {:?}-{:?} \
(with text {:?}) \
has scope {:?}",
assertion.scope_selector_text.trim(),
current_line_number, test_against_line_number, failure.column_begin, failure.column_end,
text,
scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope
);
if !out_opts.summary {
println!(" Assertion selector {:?} \
from line {:?} failed against line {:?}, column range {:?}-{:?} \
(with text {:?}) \
has scope {:?}",
assertion.scope_selector_text.trim(),
current_line_number, test_against_line_number, failure.column_begin, failure.column_end,
text,
scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope
);
}
assertion_failures += failure.column_end - failure.column_begin;
}
line_only_has_assertion = assertion.is_pure_assertion_line;
Expand All @@ -197,11 +212,11 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
test_against_line_number = current_line_number;
previous_non_assertion_line = line.to_string();
}
if debug && !line_only_has_assertion {
if out_opts.debug && !line_only_has_assertion {
println!("-- debugging line {} -- scope stack: {:?}", current_line_number, stack);
}
let ops = state.parse_line(&line);
if debug && !line_only_has_assertion {
if out_opts.debug && !line_only_has_assertion {
if ops.is_empty() && !line.is_empty() {
println!("no operations for this line...");
} else {
Expand Down Expand Up @@ -237,34 +252,46 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
}
line = line.replace("\r", &"");
}
if assertion_failures > 0 {
let res = if assertion_failures > 0 {
Ok(SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions))
} else {
Ok(SyntaxTestFileResult::Success(total_assertions))
};

if out_opts.summary {
if let Ok(SyntaxTestFileResult::FailedAssertions(failures, _)) = res {
// Don't print total assertion count so that diffs don't pick up new succeeding tests
println!("FAILED {}: {}", path.display(), failures);
}
} else {
println!("{:?}", res);
}

res
}

fn main() {
let mut args: Vec<String> = std::env::args().collect();
let debug_arg = args.iter().position(|s| s == "--debug");
if debug_arg.is_some() {
args.remove(debug_arg.unwrap());
}
let args: Vec<String> = std::env::args().collect();
let mut opts = Options::new();
opts.optflag("d", "debug", "Show parsing results for each test line");
opts.optflag("t", "time", "Time execution as a more broad-ranging benchmark");
opts.optflag("s", "summary", "Print only summary of test failures");

let time_arg = args.iter().position(|s| s == "--time");
if time_arg.is_some() {
args.remove(time_arg.unwrap());
}
let matches = match opts.parse(&args[1..]) {
Ok(m) => { m }
Err(f) => { panic!(f.to_string()) }
};

let tests_path = if args.len() < 2 {
let tests_path = if matches.free.len() < 1 {
"."
} else {
&args[1]
};
let syntaxes_path = if args.len() == 3 {
&args[2]
} else {

let syntaxes_path = if matches.free.len() < 2 {
""
} else {
&args[2]
};

// load the syntaxes from disk if told to
Expand All @@ -282,39 +309,55 @@ fn main() {
ss.link_syntaxes();
}

let exit_code = recursive_walk(&ss, &tests_path, debug_arg.is_some(), time_arg.is_some());
let out_opts = OutputOptions {
debug: matches.opt_present("debug"),
time: matches.opt_present("time"),
summary: matches.opt_present("summary"),
};

let exit_code = recursive_walk(&ss, &tests_path, out_opts);
println!("exiting with code {}", exit_code);
std::process::exit(exit_code);

}


fn recursive_walk(ss: &SyntaxSet, path: &str, debug: bool, time: bool) -> i32 {
fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 {
let mut exit_code: i32 = 0; // exit with code 0 by default, if all tests pass
let walker = WalkDir::new(path).into_iter();

// accumulate and sort for consistency of diffs across machines
let mut files = Vec::new();
for entry in walker.filter_entry(|e|e.file_type().is_dir() || is_a_syntax_test_file(e)) {
let entry = entry.unwrap();
if entry.file_type().is_file() {
println!("Testing file {}", entry.path().display());
let start = Instant::now();
let result = test_file(&ss, entry.path(), true, debug);
let elapsed = start.elapsed();
println!("{:?}", result);
if time {
let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
println!("{} ms for file {}", ms, entry.path().display());
}
if exit_code != 2 { // leave exit code 2 if there was an error
if let Err(_) = result { // set exit code 2 if there was an error
exit_code = 2;
} else if let Ok(ok) = result {
if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {
exit_code = 1; // otherwise, if there were failures, exit with code 1
}
files.push(entry.path().to_owned());
}
}
files.sort();

for path in &files {
if !out_opts.summary {
println!("Testing file {}", path.display());
}
let start = Instant::now();
let result = test_file(&ss, path, true, out_opts);
let elapsed = start.elapsed();
if out_opts.time {
let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
println!("{} ms for file {}", ms, path.display());
}
if exit_code != 2 { // leave exit code 2 if there was an error
if let Err(_) = result { // set exit code 2 if there was an error
exit_code = 2;
} else if let Ok(ok) = result {
if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {
exit_code = 1; // otherwise, if there were failures, exit with code 1
}
}
}
}

exit_code
}

Expand Down
6 changes: 6 additions & 0 deletions testdata/known_syntest_failures.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
loading syntax definitions from testdata/Packages
FAILED testdata/Packages/ASP/syntax_test_asp.asp: 162
FAILED testdata/Packages/C#/tests/syntax_test_Strings.cs: 38
FAILED testdata/Packages/LaTeX/syntax_test_latex.tex: 1
FAILED testdata/Packages/Makefile/syntax_test_makefile.mak: 7
exiting with code 1

0 comments on commit 42f3da4

Please sign in to comment.