Syntest reports

.travis.yml

@@ -34,6 +34,7 @@ script:
     fi
   - cargo test
   - make assets
+  - make syntest
   - rm -Rf examples
   - cargo doc
   # default features are required for examples to build - so remove them from sight.

Makefile

@@ -7,16 +7,25 @@ info:
 	$(info - OTHER TARGETS -------------------------------------------------------)
 	$(info themes      | generate default theme pack)
 	$(info packs       | generate default syntax pack)
-
-
+	$(info syntest     | run syntax test summary)
+
+
 $(SUBMODULES):
 	git submodule update --init --recursive
 
 assets: packs themes
 
 packs: $(SUBMODULES)
-	cargo run --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump 
-	
+	cargo run --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump
+
 themes: $(SUBMODULES)
 	cargo run --example gendata -- themepack testdata assets/default.themedump
-
+
+syntest: $(SUBMODULES)
+	@echo Tip: Run make update-known-failures to update the known failures file.
+	cargo run --release --example syntest -- testdata/Packages testdata/Packages --summary | diff -U 1000000 testdata/known_syntest_failures.txt -
+	@echo No new failures!
+
+update-known-failures: $(SUBMODULES)
+	cargo run --release --example syntest -- testdata/Packages testdata/Packages --summary | tee testdata/known_syntest_failures.txt
+

examples/syntest.rs

@@ -12,6 +12,8 @@ extern crate walkdir;
 #[macro_use]
 extern crate lazy_static;
 extern crate regex;
+extern crate getopts;
+
 //extern crate onig;
 use syntect::parsing::{SyntaxSet, ParseState, ScopeStack, Scope};
 use syntect::highlighting::ScopeSelectors;
@@ -22,9 +24,11 @@ use std::io::{BufRead, BufReader};
 use std::fs::File;
 use std::cmp::{min, max};
 use std::time::Instant;
-use walkdir::{DirEntry, WalkDir};
 use std::str::FromStr;
+
+use getopts::Options;
 use regex::Regex;
+use walkdir::{DirEntry, WalkDir};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum SyntaxTestHeaderError {
@@ -51,6 +55,13 @@ lazy_static! {
         )(.*)$"#).unwrap();
 }
 
+#[derive(Clone, Copy)]
+struct OutputOptions {
+    time: bool,
+    debug: bool,
+    summary: bool,
+}
+
 #[derive(Debug)]
 struct AssertionRange<'a> {
     begin_char: usize,
@@ -81,7 +92,7 @@ fn get_line_assertion_details<'a>(testtoken_start: &str, testtoken_end: Option<&
         if let Some(captures) = SYNTAX_TEST_ASSERTION_PATTERN.captures(&token_and_rest_of_line[testtoken_start.len()..]) {
             let mut sst = captures.get(3).unwrap().as_str(); // get the scope selector text
             let mut only_whitespace_after_token_end = true;
-            
+
             if let Some(token) = testtoken_end { // if there is an end token defined in the test file header
                 if let Some(end_token_pos) = sst.find(token) { // and there is an end token in the line
                     let (ss, after_token_end) = sst.split_at(end_token_pos); // the scope selector text ends at the end token
@@ -130,7 +141,7 @@ fn process_assertions(assertion: &AssertionRange, test_against_line_scopes: &Vec
 }
 
 /// If `parse_test_lines` is `false` then lines that only contain assertions are not parsed
-fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
+fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: OutputOptions) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
     use syntect::util::debug_print_ops;
     let f = File::open(path).unwrap();
     let mut reader = BufReader::new(f);
@@ -153,7 +164,9 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
     let syntax_file = captures.name("syntax_file").unwrap().as_str();
 
     // find the relevant syntax definition to parse the file with - case is important!
-    println!("The test file references syntax definition file: {}", syntax_file);
+    if !out_opts.summary {
+        println!("The test file references syntax definition file: {}", syntax_file);
+    }
     let syntax = ss.find_syntax_by_path(syntax_file).ok_or(SyntaxTestHeaderError::SyntaxDefinitionNotFound)?;
 
     // iterate over the lines of the file, testing them
@@ -177,15 +190,17 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
             for failure in result.iter().filter(|r|!r.success) {
                 let length = failure.column_end - failure.column_begin;
                 let text: String = previous_non_assertion_line.chars().skip(failure.column_begin).take(length).collect();
-                println!("  Assertion selector {:?} \
-                    from line {:?} failed against line {:?}, column range {:?}-{:?} \
-                    (with text {:?}) \
-                    has scope {:?}",
-                    assertion.scope_selector_text.trim(),
-                    current_line_number, test_against_line_number, failure.column_begin, failure.column_end,
-                    text,
-                    scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope
-                );
+                if !out_opts.summary {
+                    println!("  Assertion selector {:?} \
+                        from line {:?} failed against line {:?}, column range {:?}-{:?} \
+                        (with text {:?}) \
+                        has scope {:?}",
+                        assertion.scope_selector_text.trim(),
+                        current_line_number, test_against_line_number, failure.column_begin, failure.column_end,
+                        text,
+                        scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope
+                    );
+                }
                 assertion_failures += failure.column_end - failure.column_begin;
             }
             line_only_has_assertion = assertion.is_pure_assertion_line;
@@ -197,11 +212,11 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
                 test_against_line_number = current_line_number;
                 previous_non_assertion_line = line.to_string();
             }
-            if debug && !line_only_has_assertion {
+            if out_opts.debug && !line_only_has_assertion {
                 println!("-- debugging line {} -- scope stack: {:?}", current_line_number, stack);
             }
             let ops = state.parse_line(&line);
-            if debug && !line_only_has_assertion {
+            if out_opts.debug && !line_only_has_assertion {
                 if ops.is_empty() && !line.is_empty() {
                     println!("no operations for this line...");
                 } else {
@@ -237,34 +252,46 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -
         }
         line = line.replace("\r", &"");
     }
-    if assertion_failures > 0 {
+    let res = if assertion_failures > 0 {
         Ok(SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions))
     } else {
         Ok(SyntaxTestFileResult::Success(total_assertions))
+    };
+
+    if out_opts.summary {
+        if let Ok(SyntaxTestFileResult::FailedAssertions(failures, _)) = res {
+            // Don't print total assertion count so that diffs don't pick up new succeeding tests
+            println!("FAILED {}: {}", path.display(), failures);
+        }
+    } else {
+        println!("{:?}", res);
     }
+
+    res
 }
 
 fn main() {
-    let mut args: Vec<String> = std::env::args().collect();
-    let debug_arg = args.iter().position(|s| s == "--debug");
-    if debug_arg.is_some() {
-        args.remove(debug_arg.unwrap());
-    }
+    let args: Vec<String> = std::env::args().collect();
+    let mut opts = Options::new();
+    opts.optflag("d", "debug", "Show parsing results for each test line");
+    opts.optflag("t", "time", "Time execution as a more broad-ranging benchmark");
+    opts.optflag("s", "summary", "Print only summary of test failures");
 
-    let time_arg = args.iter().position(|s| s == "--time");
-    if time_arg.is_some() {
-        args.remove(time_arg.unwrap());
-    }
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => { m }
+        Err(f) => { panic!(f.to_string()) }
+    };
 
-    let tests_path = if args.len() < 2 {
+    let tests_path = if matches.free.len() < 1 {
         "."
     } else {
         &args[1]
     };
-    let syntaxes_path = if args.len() == 3 {
-        &args[2]
-    } else {
+
+    let syntaxes_path = if matches.free.len() < 2 {
         ""
+    } else {
+        &args[2]
     };
 
     // load the syntaxes from disk if told to
@@ -282,39 +309,55 @@ fn main() {
         ss.link_syntaxes();
     }
 
-    let exit_code = recursive_walk(&ss, &tests_path, debug_arg.is_some(), time_arg.is_some());
+    let out_opts = OutputOptions {
+        debug: matches.opt_present("debug"),
+        time: matches.opt_present("time"),
+        summary: matches.opt_present("summary"),
+    };
+
+    let exit_code = recursive_walk(&ss, &tests_path, out_opts);
     println!("exiting with code {}", exit_code);
     std::process::exit(exit_code);
 
 }
 
 
-fn recursive_walk(ss: &SyntaxSet, path: &str, debug: bool, time: bool) -> i32 {
+fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 {
     let mut exit_code: i32 = 0; // exit with code 0 by default, if all tests pass
     let walker = WalkDir::new(path).into_iter();
+
+    // accumulate and sort for consistency of diffs across machines
+    let mut files = Vec::new();
     for entry in walker.filter_entry(|e|e.file_type().is_dir() || is_a_syntax_test_file(e)) {
         let entry = entry.unwrap();
         if entry.file_type().is_file() {
-            println!("Testing file {}", entry.path().display());
-            let start = Instant::now();
-            let result = test_file(&ss, entry.path(), true, debug);
-            let elapsed = start.elapsed();
-            println!("{:?}", result);
-            if time {
-                let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
-                println!("{} ms for file {}", ms, entry.path().display());
-            }
-            if exit_code != 2 { // leave exit code 2 if there was an error
-                if let Err(_) = result { // set exit code 2 if there was an error
-                    exit_code = 2;
-                } else if let Ok(ok) = result {
-                    if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {
-                        exit_code = 1; // otherwise, if there were failures, exit with code 1
-                    }
+            files.push(entry.path().to_owned());
+        }
+    }
+    files.sort();
+
+    for path in &files {
+        if !out_opts.summary {
+            println!("Testing file {}", path.display());
+        }
+        let start = Instant::now();
+        let result = test_file(&ss, path, true, out_opts);
+        let elapsed = start.elapsed();
+        if out_opts.time {
+            let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
+            println!("{} ms for file {}", ms, path.display());
+        }
+        if exit_code != 2 { // leave exit code 2 if there was an error
+            if let Err(_) = result { // set exit code 2 if there was an error
+                exit_code = 2;
+            } else if let Ok(ok) = result {
+                if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {
+                    exit_code = 1; // otherwise, if there were failures, exit with code 1
                 }
             }
         }
     }
+
     exit_code
 }
 

testdata/known_syntest_failures.txt

@@ -0,0 +1,6 @@
+loading syntax definitions from testdata/Packages
+FAILED testdata/Packages/ASP/syntax_test_asp.asp: 162
+FAILED testdata/Packages/C#/tests/syntax_test_Strings.cs: 38
+FAILED testdata/Packages/LaTeX/syntax_test_latex.tex: 1
+FAILED testdata/Packages/Makefile/syntax_test_makefile.mak: 7
+exiting with code 1