From 15c028cb7a0ab9761155c3d977b7727c4c29f404 Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Thu, 25 Aug 2022 14:36:41 +0200 Subject: [PATCH 1/6] Better "unexpected kw" handling in ternary parsing --- src/kinds.jl | 15 ++++++++------- src/parser.jl | 14 ++++++++++++-- test/parser.jl | 3 +++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/kinds.jl b/src/kinds.jl index 9f461820..947fc81b 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -27,15 +27,10 @@ const _kind_names = "baremodule" "begin" "break" - "catch" "const" "continue" "do" - "else" - "elseif" - "end" "export" - "finally" "for" "function" "global" @@ -51,6 +46,13 @@ const _kind_names = "try" "using" "while" + "BEGIN_BLOCK_CONTINUATION_KEYWORDS" + "catch" + "finally" + "else" + "elseif" + "end" + "END_BLOCK_CONTINUATION_KEYWORDS" "BEGIN_CONTEXTUAL_KEYWORDS" # contextual keywords "abstract" @@ -1045,6 +1047,7 @@ end is_contextual_keyword(k::Kind) = K"BEGIN_CONTEXTUAL_KEYWORDS" < k < K"END_CONTEXTUAL_KEYWORDS" is_error(k::Kind) = K"BEGIN_ERRORS" < k < K"END_ERRORS" is_keyword(k::Kind) = K"BEGIN_KEYWORDS" < k < K"END_KEYWORDS" +is_block_continuation_keyword(k::Kind) = K"BEGIN_BLOCK_CONTINUATION_KEYWORDS" < k < K"END_BLOCK_CONTINUATION_KEYWORDS" is_literal(k::Kind) = K"BEGIN_LITERAL" < k < K"END_LITERAL" is_operator(k::Kind) = K"BEGIN_OPS" < k < K"END_OPS" is_word_operator(k::Kind) = (k == K"in" || k == K"isa" || k == K"where") @@ -1098,5 +1101,3 @@ end function is_whitespace(x) kind(x) in (K"Whitespace", K"NewlineWs") end - - diff --git a/src/parser.jl b/src/parser.jl index 2622ab14..c836a2b6 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -659,8 +659,18 @@ function parse_cond(ps::ParseState) t = peek_token(ps) if !preceding_whitespace(t) # a ? b :c ==> (if a [ ] [?] [ ] b [ ] [:] (error-t) c) - bump_invisible(ps, K"error", TRIVIA_FLAG, - error="space required after `:` in `?` expression") + bump_invisible(ps, K"error", TRIVIA_FLAG, error="space required after `:` in `?` expression") + end + if is_block_continuation_keyword(kind(t)) + # a "continuaton keyword" is likely to belong to the surrounding code, so + # we abort early + + # if true; x ? true elseif true end ==> (if true (block (if x true (error-t) (error-t))) (elseif true (block))) + # if true; x ? true end ==> (if true (block (if x true (error-t) (error-t)))) + # if true; x ? true : elseif true end ==> (if true (block (if x true (error-t))) (elseif true (block))) + bump_invisible(ps, K"error", TRIVIA_FLAG, error="unexpected `$(kind(t))`") + emit(ps, mark, K"if") + return end parse_eq_star(ps) emit(ps, mark, K"if") diff --git a/test/parser.jl b/test/parser.jl index 41be1841..3771e8cd 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -402,6 +402,9 @@ tests = [ "if a xx elseif b yy end" => "(if a (block xx) (elseif b (block yy)))" "if a xx else if b yy end" => "(if a (block xx) (error-t) (elseif b (block yy)))" "if a xx else yy end" => "(if a (block xx) (block yy))" + "if true; x ? true elseif true end" => "(if true (block (if x true (error-t) (error-t))) (elseif true (block)))" + "if true; x ? true end" => "(if true (block (if x true (error-t) (error-t))))" + "if true; x ? true : elseif true end" => "(if true (block (if x true (error-t))) (elseif true (block)))" ], JuliaSyntax.parse_const_local_global => [ "global x" => "(global x)" From 62d6226854d1a264456c2c0e5b8b050bc7913f17 Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Tue, 30 Aug 2022 10:55:07 +0200 Subject: [PATCH 2/6] Also handle newline ws --- src/parser.jl | 3 ++- test/parser.jl | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/parser.jl b/src/parser.jl index 3f393241..8edcbf4c 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -637,7 +637,7 @@ function parse_cond(ps::ParseState) # a ? b c ==> (if a b (error) c) bump_invisible(ps, K"error", TRIVIA_FLAG, error="`:` expected in `?` expression") end - t = peek_token(ps) + t = peek_token(ps; skip_newlines = true) if !preceding_whitespace(t) # a ? b :c ==> (if a [ ] [?] [ ] b [ ] [:] (error-t) c) bump_invisible(ps, K"error", TRIVIA_FLAG, error="space required after `:` in `?` expression") @@ -648,6 +648,7 @@ function parse_cond(ps::ParseState) # if true; x ? true elseif true end ==> (if true (block (if x true (error-t) (error-t))) (elseif true (block))) # if true; x ? true end ==> (if true (block (if x true (error-t) (error-t)))) + # if true; x ? true\n end ==> (if true (block (if x true (error-t) (error-t)))) # if true; x ? true : elseif true end ==> (if true (block (if x true (error-t))) (elseif true (block))) bump_invisible(ps, K"error", TRIVIA_FLAG, error="unexpected `$(kind(t))`") emit(ps, mark, K"if") diff --git a/test/parser.jl b/test/parser.jl index 0758cc60..dd667a60 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -404,6 +404,7 @@ tests = [ "if a xx else yy end" => "(if a (block xx) (block yy))" "if true; x ? true elseif true end" => "(if true (block (if x true (error-t) (error-t))) (elseif true (block)))" "if true; x ? true end" => "(if true (block (if x true (error-t) (error-t))))" + "if true; x ? true\nend" => "(if true (block (if x true (error-t) (error-t))))" "if true; x ? true : elseif true end" => "(if true (block (if x true (error-t))) (elseif true (block)))" ], JuliaSyntax.parse_const_local_global => [ From 7b6ad2787389b2e0c762ee04de83d477fa19600d Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Tue, 30 Aug 2022 09:38:42 +0000 Subject: [PATCH 3/6] improve diagnostics script --- .gitignore | 2 ++ tools/check_all_packages.jl | 64 +++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index b067edde..e3f57ade 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /Manifest.toml +/tools/pkgs +/tools/logs.txt \ No newline at end of file diff --git a/tools/check_all_packages.jl b/tools/check_all_packages.jl index 2a52a947..29e32922 100644 --- a/tools/check_all_packages.jl +++ b/tools/check_all_packages.jl @@ -5,6 +5,23 @@ using JuliaSyntax, Logging +# like Meta.parseall, but throws +function parseall(str) + pos = firstindex(str) + exs = [] + while pos <= lastindex(str) + ex, pos = Meta.parse(str, pos) + push!(exs, ex) + end + if length(exs) == 0 + throw(Meta.ParseError("end of input")) + elseif length(exs) == 1 + return exs[1] + else + return Expr(:toplevel, exs...) + end +end + logio = open(joinpath(@__DIR__, "logs.txt"), "w") logger = Logging.ConsoleLogger(logio) @@ -35,18 +52,44 @@ Logging.with_logger(logger) do t = time() i = 0 iob = IOBuffer() + ex_count = 0 for (r, _, files) in walkdir(pkgspath) for f in files endswith(f, ".jl") || continue fpath = joinpath(r, f) - try - JuliaSyntax.parse(Expr, read(fpath, String)) - catch err - err isa InterruptException && rethrow() - ex = (err, catch_backtrace()) - push!(exceptions, ex) - @error "parsing failed for $(fpath)" ex - flush(logio) + if isfile(fpath) + file = read(fpath, String) + try + e1 = JuliaSyntax.parse(Expr, file) + catch err + err isa InterruptException && rethrow() + ex_count += 1 + ex = (err, catch_backtrace()) + push!(exceptions, ex) + meta_parse = "success" + try + parseall(file) + catch err2 + meta_parse = "fail" + ex_count -= 1 + end + parse_to_syntax = "success" + try + JuliaSyntax.parse(JuliaSyntax.SyntaxNode, file) + catch err2 + parse_to_syntax = "fail" + end + severity = parse_to_syntax == "fail" ? "error" : + meta_parse == "fail" ? "warn" : "error" + println(logio, """ + [$(severity)] $(fpath) + parse-to-expr: fail + parse-to-syntaxtree: $(parse_to_syntax) + reference: $(meta_parse) + """) + @error "" exception = ex + flush(logio) + end end i += 1 if i % 100 == 0 @@ -54,8 +97,9 @@ Logging.with_logger(logger) do avg = round(runtime/i*1000, digits = 2) print(iob, "\e[2J\e[0;0H") println(iob, "$i files parsed") - println(iob, " $(length(exceptions)) failures") - println(iob, " $(avg)ms per file, $(round(Int, runtime))s in total") + println(iob, "> $(ex_count) failures compared to Meta.parse") + println(iob, "> $(length(exceptions)) errors in total") + println(iob, "> $(avg)ms per file, $(round(Int, runtime))s in total") println(stderr, String(take!(iob))) end end From d8cd0659dbd8af5f8fb6aa9dc9503d809fc0f8b1 Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Wed, 31 Aug 2022 11:25:55 +0200 Subject: [PATCH 4/6] Add FIXME --- src/parser.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/parser.jl b/src/parser.jl index 8edcbf4c..8f232df5 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -642,6 +642,9 @@ function parse_cond(ps::ParseState) # a ? b :c ==> (if a [ ] [?] [ ] b [ ] [:] (error-t) c) bump_invisible(ps, K"error", TRIVIA_FLAG, error="space required after `:` in `?` expression") end + + # FIXME: This is a very specific case. Error recovery should be handled mor + # generally elsewhere. if is_block_continuation_keyword(kind(t)) # a "continuaton keyword" is likely to belong to the surrounding code, so # we abort early From dafe88b85ea693be02bcf6c0b11c5141d7aa2f5c Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Wed, 31 Aug 2022 11:33:05 +0200 Subject: [PATCH 5/6] Update src/parser.jl Co-authored-by: Kristoffer Carlsson --- src/parser.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.jl b/src/parser.jl index 8f232df5..b943a397 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -643,7 +643,7 @@ function parse_cond(ps::ParseState) bump_invisible(ps, K"error", TRIVIA_FLAG, error="space required after `:` in `?` expression") end - # FIXME: This is a very specific case. Error recovery should be handled mor + # FIXME: This is a very specific case. Error recovery should be handled more # generally elsewhere. if is_block_continuation_keyword(kind(t)) # a "continuaton keyword" is likely to belong to the surrounding code, so From c67f95c58fa30611a6cd9ded8b114572df14b53d Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Wed, 14 Sep 2022 12:46:48 +0200 Subject: [PATCH 6/6] rename parseall to parseall_throws --- tools/check_all_packages.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/check_all_packages.jl b/tools/check_all_packages.jl index 29e32922..40f55bfa 100644 --- a/tools/check_all_packages.jl +++ b/tools/check_all_packages.jl @@ -6,7 +6,7 @@ using JuliaSyntax, Logging # like Meta.parseall, but throws -function parseall(str) +function parseall_throws(str) pos = firstindex(str) exs = [] while pos <= lastindex(str) @@ -68,7 +68,7 @@ Logging.with_logger(logger) do push!(exceptions, ex) meta_parse = "success" try - parseall(file) + parseall_throws(file) catch err2 meta_parse = "fail" ex_count -= 1