Merge pull request #77 from JuliaLang/sp/ternary-unexpected-kw

Better "unexpected kw" handling in ternary parsing

.gitignore

@@ -1 +1,3 @@
 /Manifest.toml
+/tools/pkgs
+/tools/logs.txt

src/kinds.jl

@@ -27,15 +27,10 @@ const _kind_names =
         "baremodule"
         "begin"
         "break"
-        "catch"
         "const"
         "continue"
         "do"
-        "else"
-        "elseif"
-        "end"
         "export"
-        "finally"
         "for"
         "function"
         "global"
@@ -51,6 +46,13 @@ const _kind_names =
         "try"
         "using"
         "while"
+        "BEGIN_BLOCK_CONTINUATION_KEYWORDS"
+            "catch"
+            "finally"
+            "else"
+            "elseif"
+            "end"
+        "END_BLOCK_CONTINUATION_KEYWORDS"
         "BEGIN_CONTEXTUAL_KEYWORDS"
             # contextual keywords
             "abstract"
@@ -1046,6 +1048,7 @@ end
 is_contextual_keyword(k::Kind) = K"BEGIN_CONTEXTUAL_KEYWORDS" < k < K"END_CONTEXTUAL_KEYWORDS"
 is_error(k::Kind) = K"BEGIN_ERRORS" < k < K"END_ERRORS"
 is_keyword(k::Kind) = K"BEGIN_KEYWORDS" < k < K"END_KEYWORDS"
+is_block_continuation_keyword(k::Kind) = K"BEGIN_BLOCK_CONTINUATION_KEYWORDS" < k < K"END_BLOCK_CONTINUATION_KEYWORDS"
 is_literal(k::Kind) = K"BEGIN_LITERAL" < k < K"END_LITERAL"
 is_operator(k::Kind) = K"BEGIN_OPS" < k < K"END_OPS"
 is_word_operator(k::Kind) = (k == K"in" || k == K"isa" || k == K"where")
@@ -1099,5 +1102,3 @@ end
 function is_whitespace(x)
     kind(x) in (K"Whitespace", K"NewlineWs")
 end
-
-

src/parser.jl

@@ -637,12 +637,27 @@ function parse_cond(ps::ParseState)
         # a ? b c  ==>  (? a b (error-t) c)
         bump_invisible(ps, K"error", TRIVIA_FLAG, error="`:` expected in `?` expression")
     end
-    t = peek_token(ps)
+    t = peek_token(ps; skip_newlines = true)
     if !preceding_whitespace(t)
         # a ? b :c  ==>  (? a b (error-t) c)
         bump_invisible(ps, K"error", TRIVIA_FLAG,
                        error="space required after `:` in `?` expression")
     end
+
+    # FIXME: This is a very specific case. Error recovery should be handled more
+    # generally elsewhere.
+    if is_block_continuation_keyword(kind(t))
+        # a "continuaton keyword" is likely to belong to the surrounding code, so
+        # we abort early
+
+        # if true; x ? true elseif true end  ==> (if true (block (if x true (error-t) (error-t))) (elseif true (block)))
+        # if true; x ? true end  ==> (if true (block (if x true (error-t) (error-t))))
+        # if true; x ? true\n end  ==> (if true (block (if x true (error-t) (error-t))))
+        # if true; x ? true : elseif true end  ==> (if true (block (if x true (error-t))) (elseif true (block)))
+        bump_invisible(ps, K"error", TRIVIA_FLAG, error="unexpected `$(kind(t))`")
+        emit(ps, mark, K"if")
+        return
+    end
     parse_eq_star(ps)
     emit(ps, mark, K"?")
 end

test/parser.jl

@@ -403,6 +403,10 @@ tests = [
         "if a xx elseif b yy end"   =>  "(if a (block xx) (elseif b (block yy)))"
         "if a xx else if b yy end"  =>  "(if a (block xx) (error-t) (elseif b (block yy)))"
         "if a xx else yy end"   =>  "(if a (block xx) (block yy))"
+        "if true; x ? true elseif true end"  => "(if true (block (if x true (error-t) (error-t))) (elseif true (block)))"
+        "if true; x ? true end"  => "(if true (block (if x true (error-t) (error-t))))"
+        "if true; x ? true\nend"  => "(if true (block (if x true (error-t) (error-t))))"
+        "if true; x ? true : elseif true end"  => "(if true (block (if x true (error-t))) (elseif true (block)))"
     ],
     JuliaSyntax.parse_const_local_global => [
         "global x"             =>  "(global x)"

tools/check_all_packages.jl

@@ -5,6 +5,23 @@
 
 using JuliaSyntax, Logging
 
+# like Meta.parseall, but throws
+function parseall_throws(str)
+    pos = firstindex(str)
+    exs = []
+    while pos <= lastindex(str)
+        ex, pos = Meta.parse(str, pos)
+        push!(exs, ex)
+    end
+    if length(exs) == 0
+        throw(Meta.ParseError("end of input"))
+    elseif length(exs) == 1
+        return exs[1]
+    else
+        return Expr(:toplevel, exs...)
+    end
+end
+
 logio = open(joinpath(@__DIR__, "logs.txt"), "w")
 logger = Logging.ConsoleLogger(logio)
 
@@ -35,27 +52,54 @@ Logging.with_logger(logger) do
     t = time()
     i = 0
     iob = IOBuffer()
+    ex_count = 0
     for (r, _, files) in walkdir(pkgspath)
         for f in files
             endswith(f, ".jl") || continue
             fpath = joinpath(r, f)
-            try
-                JuliaSyntax.parse(Expr, read(fpath, String))
-            catch err
-                err isa InterruptException && rethrow()
-                ex = (err, catch_backtrace())
-                push!(exceptions, ex)
-                @error "parsing failed for $(fpath)" ex
-                flush(logio)
+            if isfile(fpath)
+                file = read(fpath, String)
+                try
+                    e1 = JuliaSyntax.parse(Expr, file)
+                catch err
+                    err isa InterruptException && rethrow()
+                    ex_count += 1
+                    ex = (err, catch_backtrace())
+                    push!(exceptions, ex)
+                    meta_parse = "success"
+                    try
+                        parseall_throws(file)
+                    catch err2
+                        meta_parse = "fail"
+                        ex_count -= 1
+                    end
+                    parse_to_syntax = "success"
+                    try
+                        JuliaSyntax.parse(JuliaSyntax.SyntaxNode, file)
+                    catch err2
+                        parse_to_syntax = "fail"
+                    end
+                    severity = parse_to_syntax == "fail" ? "error" :
+                        meta_parse == "fail" ? "warn" : "error"
+                    println(logio, """
+                    [$(severity)] $(fpath)
+                      parse-to-expr: fail
+                      parse-to-syntaxtree: $(parse_to_syntax)
+                      reference: $(meta_parse)
+                    """)
+                    @error "" exception = ex
+                    flush(logio)
+                end
             end
             i += 1
             if i % 100 == 0
                 runtime = time() - t
                 avg = round(runtime/i*1000, digits = 2)
                 print(iob, "\e[2J\e[0;0H")
                 println(iob, "$i files parsed")
-                println(iob, "  $(length(exceptions)) failures")
-                println(iob, "  $(avg)ms per file, $(round(Int, runtime))s in total")
+                println(iob, "> $(ex_count) failures compared to Meta.parse")
+                println(iob, "> $(length(exceptions)) errors in total")
+                println(iob, "> $(avg)ms per file, $(round(Int, runtime))s in total")
                 println(stderr, String(take!(iob)))
             end
         end