diff --git a/test/test_utils.jl b/test/test_utils.jl index 2059f96a..c0b1e7ab 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -245,14 +245,7 @@ function equals_flisp_parse(exprs_equal, tree) exprs_equal(fl_ex, ex) end -""" - reduce_test(text::AbstractString; exprs_equal=exprs_equal_no_linenum) - reduce_test(tree::SyntaxNode; exprs_equal=exprs_equal_no_linenum) - -Select minimal subtrees of `text` or `tree` which are inconsistent between -flisp and JuliaSyntax parsers. -""" -function reduce_test(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum) +function _reduce_test(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum) if equals_flisp_parse(exprs_equal, tree) return false end @@ -266,7 +259,7 @@ function reduce_test(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum) if is_trivia(child) || !haschildren(child) continue end - had_failing_subtrees |= reduce_test(failing_subtrees, child; exprs_equal=exprs_equal) + had_failing_subtrees |= _reduce_test(failing_subtrees, child; exprs_equal=exprs_equal) end end if !had_failing_subtrees @@ -275,9 +268,16 @@ function reduce_test(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum) return true end +""" + reduce_test(text::AbstractString; exprs_equal=exprs_equal_no_linenum) + reduce_test(tree::SyntaxNode; exprs_equal=exprs_equal_no_linenum) + +Select minimal subtrees of `text` or `tree` which are inconsistent between +flisp and JuliaSyntax parsers. +""" function reduce_test(tree::SyntaxNode; kws...) subtrees = Vector{typeof(tree)}() - reduce_test(subtrees, tree; kws...) + _reduce_test(subtrees, tree; kws...) subtrees end diff --git a/tools/check_all_packages.jl b/tools/check_all_packages.jl index 3ee160c0..5e9f47b5 100644 --- a/tools/check_all_packages.jl +++ b/tools/check_all_packages.jl @@ -3,76 +3,69 @@ # # Run this after registry_download.jl (so the pkgs directory is populated). -using JuliaSyntax, Logging, Serialization +using JuliaSyntax, Logging, TerminalLoggers, ProgressLogging, Serialization include("../test/test_utils.jl") -logio = open(joinpath(@__DIR__, "logs.txt"), "w") -logger = Logging.ConsoleLogger(logio) - pkgspath = joinpath(@__DIR__, "pkgs") +source_paths = find_source_in_path(pkgspath) +file_count = length(source_paths) exception_count = 0 mismatch_count = 0 -file_count = 0 t0 = time() exceptions = [] -Logging.with_logger(logger) do - global exception_count, mismatch_count, file_count, t0 - for (r, _, files) in walkdir(pkgspath) - for f in files - endswith(f, ".jl") || continue - fpath = joinpath(r, f) - isfile(fpath) || continue - - code = read(fpath, String) - expr_cache = fpath*".Expr" - #e2 = JuliaSyntax.fl_parseall(code) - e2 = open(deserialize, fpath*".Expr") - @assert Meta.isexpr(e2, :toplevel) - try - e1 = JuliaSyntax.parseall(Expr, code, filename=fpath, ignore_warnings=true) - if !exprs_roughly_equal(e2, e1) - mismatch_count += 1 - @error("Parsers succeed but disagree", - fpath, - diff=Text(sprint(show_expr_text_diff, show, e1, e2)), - ) - end - catch err - err isa InterruptException && rethrow() - ex = (err, catch_backtrace()) - push!(exceptions, ex) - ref_parse = "success" - if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete)) - ref_parse = "fail" - if err isa JuliaSyntax.ParseError - # Both parsers agree that there's an error, and - # JuliaSyntax didn't have an internal error. - continue +Logging.with_logger(TerminalLogger()) do + global exception_count, mismatch_count, t0 + @withprogress for (ifile, fpath) in enumerate(source_paths) + @logprogress ifile/file_count time_ms=round((time() - t0)/ifile*1000, digits = 2) + text = read(fpath, String) + expr_cache = fpath*".Expr" + #e2 = JuliaSyntax.fl_parseall(text) + e2 = open(deserialize, fpath*".Expr") + @assert Meta.isexpr(e2, :toplevel) + try + e1 = JuliaSyntax.parseall(Expr, text, filename=fpath, ignore_warnings=true) + if !exprs_roughly_equal(e2, e1) + mismatch_count += 1 + reduced_chunks = sprint(context=:color=>true) do io + for c in reduce_test(text) + JuliaSyntax.highlight(io, c.source, range(c), context_inner_lines=5) + println(io, "\n") end end - - exception_count += 1 - parse_to_syntax = "success" - try - JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, code) - catch err2 - parse_to_syntax = "fail" + @error("Parsers succeed but disagree", + fpath, + reduced_chunks=Text(reduced_chunks), + # diff=Text(sprint(show_expr_text_diff, show, e1, e2)), + ) + end + catch err + err isa InterruptException && rethrow() + ex = (err, catch_backtrace()) + push!(exceptions, ex) + ref_parse = "success" + if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete)) + ref_parse = "fail" + if err isa JuliaSyntax.ParseError + # Both parsers agree that there's an error, and + # JuliaSyntax didn't have an internal error. + continue end - @error "Parse failed" fpath exception=ex parse_to_syntax end - file_count += 1 - if file_count % 100 == 0 - t_avg = round((time() - t0)/file_count*1000, digits = 2) - print(stderr, "\r$file_count files parsed, $t_avg ms per file") + exception_count += 1 + parse_to_syntax = "success" + try + JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, code) + catch err2 + parse_to_syntax = "fail" end + @error "Parse failed" fpath exception=ex parse_to_syntax end end end -close(logio) t_avg = round((time() - t0)/file_count*1000, digits = 2)