From 70c908f0e5c3140039aaa4ec873e64c86c350fa9 Mon Sep 17 00:00:00 2001 From: c42f Date: Tue, 21 Mar 2023 06:49:49 +1000 Subject: [PATCH] Enclose grouping parentheses with `parens` node (#222) Introduce a new kind `K"parens"` to represent grouping parentheses with a tree node of their own. This makes it simple for tooling to process and preserve parenthesized expressions without resorting to searching through the attached syntax trivia. An alternative considered here was to use `K"block"` with a single child which would avoid introducing an extra kind of node. But in that case we couldn't distinguish between a trivial block like `(a;)` vs bare parentheses `(a)`. It also makes implementing `peek_behind` more complicated. --- src/expr.jl | 3 ++ src/parse_stream.jl | 37 +++++++------- src/parser.jl | 105 ++++++++++++++++++++------------------- test/parser.jl | 117 +++++++++++++++++++++++--------------------- test/parser_api.jl | 6 +-- 5 files changed, 138 insertions(+), 130 deletions(-) diff --git a/src/expr.jl b/src/expr.jl index b18b9c0e..2f6ee29e 100644 --- a/src/expr.jl +++ b/src/expr.jl @@ -197,6 +197,9 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true, end elseif headsym === :where reorder_parameters!(args, 2) + elseif headsym == :parens + # parens are used for grouping and don't appear in the Expr AST + return only(args) elseif headsym in (:try, :try_finally_catch) # Try children in source order: # try_block catch_var catch_block else_block finally_block diff --git a/src/parse_stream.jl b/src/parse_stream.jl index 8db4489f..0ff02159 100644 --- a/src/parse_stream.jl +++ b/src/parse_stream.jl @@ -582,26 +582,25 @@ function first_child_position(stream::ParseStream, pos::ParseStreamPosition) end function peek_behind(stream::ParseStream; skip_trivia::Bool=true) - pos = position(stream) - if !skip_trivia || !token_is_last(stream, pos) - return peek_behind(stream, pos) - else - token_index = lastindex(stream.tokens) - range_index = lastindex(stream.ranges) - last_token_in_nonterminal = isempty(stream.ranges) ? 0 : - stream.ranges[range_index].last_token - while token_index > last_token_in_nonterminal - t = stream.tokens[token_index] - if !is_trivia(t) && kind(t) != K"TOMBSTONE" - break - end - token_index -= 1 - end - if token_index > 0 - return peek_behind(stream, ParseStreamPosition(token_index, range_index)) - else - internal_error("Can't peek behind at start of stream") + token_index = lastindex(stream.tokens) + range_index = lastindex(stream.ranges) + while range_index >= firstindex(stream.ranges) && + kind(stream.ranges[range_index]) == K"parens" + range_index -= 1 + end + last_token_in_nonterminal = range_index == 0 ? 0 : + stream.ranges[range_index].last_token + while token_index > last_token_in_nonterminal + t = stream.tokens[token_index] + if kind(t) != K"TOMBSTONE" && (!skip_trivia || !is_trivia(t)) + break end + token_index -= 1 + end + if token_index > 0 + return peek_behind(stream, ParseStreamPosition(token_index, range_index)) + else + internal_error("Can't peek behind at start of stream") end end diff --git a/src/parser.jl b/src/parser.jl index 46a51bf0..89936013 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -315,8 +315,8 @@ function was_eventually_call(ps::ParseState) b = peek_behind(stream, p) if b.kind == K"call" return true - elseif b.kind == K"where" || (b.kind == K"::" && - has_flags(b.flags, INFIX_FLAG)) + elseif b.kind == K"where" || b.kind == K"parens" || + (b.kind == K"::" && has_flags(b.flags, INFIX_FLAG)) p = first_child_position(ps, p) else return false @@ -885,7 +885,7 @@ function parse_range(ps::ParseState) if had_newline # Error message for people coming from python # 1:\n2 ==> (call-i 1 : (error)) - # (1:\n2) ==> (call-i 1 : 2) + # (1:\n2) ==> (parens (call-i 1 : 2)) emit_diagnostic(ps, whitespace=true, error="line break after `:` in range expression") bump_invisible(ps, K"error") @@ -1021,7 +1021,7 @@ function parse_unary_subtype(ps::ParseState) elseif k2 in KSet"{ (" # parse <:{T}(x::T) or <:(x::T) like other unary operators # <:{T}(x::T) ==> (call (curly <: T) (:: x T)) - # <:(x::T) ==> (<:-pre (:: x T)) + # <:(x::T) ==> (<:-pre (parens (:: x T))) parse_where(ps, parse_juxtapose) else # <: x ==> (<:-pre x) @@ -1108,9 +1108,9 @@ end # Juxtoposition. Ugh! But so useful for units and Field identities like `im` # # 2x ==> (juxtapose 2 x) -# 2(x) ==> (juxtapose 2 x) -# (2)(3)x ==> (juxtapose 2 3 x) -# (x-1)y ==> (juxtapose (call-i x - 1) y) +# 2(x) ==> (juxtapose 2 (parens x)) +# (2)(3)x ==> (juxtapose (parens 2) (parens 3) x) +# (x-1)y ==> (juxtapose (parens (call-i x - 1)) y) # x'y ==> (juxtapose (call-post x ') y) # # flisp: parse-juxtapose @@ -1239,9 +1239,9 @@ function parse_unary(ps::ParseState) mark_before_paren = position(ps) bump(ps, TRIVIA_FLAG) # ( - initial_semi = peek(ps) == K";" + _is_paren_call = peek(ps, skip_newlines=true) in KSet"; )" opts = parse_brackets(ps, K")") do had_commas, had_splat, num_semis, num_subexprs - is_paren_call = had_commas || had_splat || initial_semi + is_paren_call = had_commas || had_splat || _is_paren_call return (needs_parameters=is_paren_call, is_paren_call=is_paren_call, is_block=!is_paren_call && num_semis > 0) @@ -1263,6 +1263,7 @@ function parse_unary(ps::ParseState) # +(a...) ==> (call + (... a)) # +(a;b,c) ==> (call + a (parameters b c)) # +(;a) ==> (call + (parameters a)) + # +() ==> (call +) # Prefix calls have higher precedence than ^ # +(a,b)^2 ==> (call-i (call + a b) ^ 2) # +(a,b)(x)^2 ==> (call-i (call (call + a b) x) ^ 2) @@ -1292,21 +1293,23 @@ function parse_unary(ps::ParseState) parse_factor_with_initial_ex(ps, mark) else # Unary function calls with brackets as grouping, not an arglist - # .+(a) ==> (dotcall-pre (. +) a) + # .+(a) ==> (dotcall-pre (. +) (parens a)) if opts.is_block # +(a;b) ==> (call-pre + (block-p a b)) emit(ps, mark_before_paren, K"block", PARENS_FLAG) + else + emit(ps, mark_before_paren, K"parens") end # Not a prefix operator call but a block; `=` is not `kw` - # +(a=1) ==> (call-pre + (= a 1)) + # +(a=1) ==> (call-pre + (parens (= a 1))) # Unary operators have lower precedence than ^ - # +(a)^2 ==> (call-pre + (call-i a ^ 2)) - # .+(a)^2 ==> (dotcall-pre + (call-i a ^ 2)) - # +(a)(x,y)^2 ==> (call-pre + (call-i (call a x y) ^ 2)) + # +(a)^2 ==> (call-pre + (call-i (parens a) ^ 2)) + # .+(a)^2 ==> (dotcall-pre + (call-i (parens a) ^ 2)) + # +(a)(x,y)^2 ==> (call-pre + (call-i (call (parens a) x y) ^ 2)) parse_call_chain(ps, mark_before_paren) parse_factor_with_initial_ex(ps, mark_before_paren) if is_type_operator(op_t) - # <:(a) ==> (<:-pre a) + # <:(a) ==> (<:-pre (parens a)) emit(ps, mark, op_k, PREFIX_OP_FLAG) reset_node!(ps, op_pos, flags=TRIVIA_FLAG) else @@ -1451,7 +1454,7 @@ function parse_identifier_or_interpolate(ps::ParseState) mark = position(ps) parse_unary_prefix(ps) b = peek_behind(ps) - # export (x::T) ==> (export (error (::-i x T))) + # export (x::T) ==> (export (error (parens (::-i x T)))) # export outer ==> (export outer) # export ($f) ==> (export ($ f)) ok = (b.is_leaf && (b.kind == K"Identifier" || is_operator(b.kind))) || @@ -1491,13 +1494,13 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) k = kind(t) if !is_macrocall && ps.space_sensitive && preceding_whitespace(t) && k in KSet"( [ { \" \"\"\" ` ```" - # [f (x)] ==> (hcat f x) + # [f (x)] ==> (hcat f (parens x)) # [f x] ==> (hcat f x) break elseif is_macrocall && (preceding_whitespace(t) || !(k in KSet"( [ { ' .")) # Macro calls with space-separated arguments # @foo a b ==> (macrocall @foo a b) - # @foo (x) ==> (macrocall @foo x) + # @foo (x) ==> (macrocall @foo (parens x)) # @foo (x,y) ==> (macrocall @foo (tuple-p x y)) # [@foo x] ==> (vect (macrocall @foo x)) # [@foo] ==> (vect (macrocall @foo)) @@ -1537,8 +1540,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) # f(a,b) ==> (call f a b) # f(a=1; b=2) ==> (call f (= a 1) (parameters (= b 2))) # f(a; b; c) ==> (call f a (parameters b) (parameters c)) - # (a=1)() ==> (call (= a 1)) - # f (a) ==> (call f (error-t) a b) + # (a=1)() ==> (call (parens (= a 1))) + # f (a) ==> (call f (error-t) a) bump_disallowed_space(ps) bump(ps, TRIVIA_FLAG) parse_call_arglist(ps, K")") @@ -1580,7 +1583,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) else # a[i] ==> (ref a i) # a[i,j] ==> (ref a i j) - # (a=1)[] ==> (ref (= a 1)) + # (a=1)[] ==> (ref (parens (= a 1))) + # a[end] ==> (ref a end) # T[x y] ==> (typed_hcat T x y) # T[x ; y] ==> (typed_vcat T x y) # T[a b; c d] ==> (typed_vcat T (row a b) (row c d)) @@ -1905,7 +1909,7 @@ function parse_resword(ps::ParseState) else # Function/macro definition with no methods # function f end ==> (function f) - # (function f \n end) ==> (function f) + # (function f \n end) ==> (parens (function f)) # function f \n\n end ==> (function f) # function $f end ==> (function ($ f)) # macro f end ==> (macro f) @@ -2007,7 +2011,7 @@ function parse_resword(ps::ParseState) # export a, \n @b ==> (export a @b) # export +, == ==> (export + ==) # export \n a ==> (export a) - # export \$a, \$(a*b) ==> (export (\$ a) (\$ (call-i a * b))) + # export \$a, \$(a*b) ==> (export (\$ a) (\$ (parens (call-i a * b)))) bump(ps, TRIVIA_FLAG) parse_comma_separated(ps, parse_atsym) emit(ps, mark, K"export") @@ -2105,10 +2109,10 @@ function parse_function_signature(ps::ParseState, is_function::Bool) emit(ps, mark, K"error", error="Invalid macro name") else # macro f() end ==> (macro (call f) (block)) - # macro (:)(ex) end ==> (macro (call : ex) (block)) - # macro (type)(ex) end ==> (macro (call type ex) (block)) + # macro (:)(ex) end ==> (macro (call (parens :) ex) (block)) + # macro (type)(ex) end ==> (macro (call (parens type) ex) (block)) # macro $f() end ==> (macro (call ($ f)) (block)) - # macro ($f)() end ==> (macro (call ($ f)) (block)) + # macro ($f)() end ==> (macro (call (parens ($ f))) (block)) end else if peek(ps) == K"(" @@ -2145,10 +2149,11 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function ()(x) end ==> (function (call (tuple-p) x) (block)) emit(ps, mark, K"tuple", PARENS_FLAG) else - # function (A).f() end ==> (function (call (. A (quote f))) (block)) - # function (:)() end ==> (function (call :) (block)) - # function (x::T)() end ==> (function (call (::-i x T)) (block)) - # function (::T)() end ==> (function (call (::-pre T)) (block)) + # function (A).f() end ==> (function (call (. (parens A) (quote f))) (block)) + # function (:)() end ==> (function (call (parens :)) (block)) + # function (x::T)() end ==> (function (call (parens (::-i x T))) (block)) + # function (::T)() end ==> (function (call (parens (::-pre T))) (block)) + emit(ps, mark, K"parens", PARENS_FLAG) end else parse_unary_prefix(ps) @@ -2163,8 +2168,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function type() end ==> (function (call type) (block)) # function \n f() end ==> (function (call f) (block)) # function $f() end ==> (function (call ($ f)) (block)) - # function (:)() end ==> (function (call :) (block)) - # function (::Type{T})(x) end ==> (function (call (::-pre (curly Type T)) x) (block)) + # function (::Type{T})(x) end ==> (function (call (parens (::-pre (curly Type T))) x) (block)) end end end @@ -2205,8 +2209,8 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function (f() where T) end ==> (function (where (call f) T) (block)) # function (f()) where T end ==> (function (where (call f) T) (block)) # function (f() where T) where U end ==> (function (where (where (call f) T) U) (block)) - # function (f()::S) end ==> (function (::-i (call f) S) (block)) - # function ((f()::S) where T) end ==> (function (where (::-i (call f) S) T) (block)) + # function (f()::S) end ==> (function (parens (::-i (call f) S)) (block)) + # function ((f()::S) where T) end ==> (function (where (parens (::-i (call f) S)) T) (block)) # # TODO: Warn for use of parens? The precedence of `::` and # `where` don't work inside parens so this is a bit of a syntax @@ -2401,7 +2405,7 @@ function parse_atsym(ps::ParseState) else # export a ==> (export a) # export \n a ==> (export a) - # export $a, $(a*b) ==> (export ($ a) ($ (call * a b))) + # export $a, $(a*b) ==> (export ($ a) (parens ($ (call * a b)))) parse_identifier_or_interpolate(ps) end end @@ -2706,7 +2710,7 @@ end function parse_generator(ps::ParseState, mark, flatten=false) t = peek_token(ps) if !preceding_whitespace(t) - # [(x)for x in xs] ==> (comprehension (generator x (error) (= x xs))) + # [(x)for x in xs] ==> (comprehension (generator (parens x) (error) (= x xs))) bump_invisible(ps, K"error", TRIVIA_FLAG, error="Expected space before `for` in generator") end @@ -2715,21 +2719,21 @@ function parse_generator(ps::ParseState, mark, flatten=false) filter_mark = position(ps) parse_comma_separated(ps, parse_iteration_spec) if peek(ps) == K"if" - # (a for x in xs if cond) ==> (generator a (filter (= x xs) cond)) + # (a for x in xs if cond) ==> (parens (generator a (filter (= x xs) cond))) bump(ps, TRIVIA_FLAG) parse_cond(ps) emit(ps, filter_mark, K"filter") end t = peek_token(ps) if kind(t) == K"for" - # (xy for x in xs for y in ys) ==> (flatten xy (= x xs) (= y ys)) - # (xy for x in xs for y in ys for z in zs) ==> (flatten xy (= x xs) (= y ys) (= z zs)) + # (xy for x in xs for y in ys) ==> (parens (flatten xy (= x xs) (= y ys))) + # (xy for x in xs for y in ys for z in zs) ==> (parens (flatten xy (= x xs) (= y ys) (= z zs))) parse_generator(ps, mark, true) if !flatten emit(ps, mark, K"flatten") end elseif !flatten - # (x for a in as) ==> (generator x (= a as)) + # (x for a in as) ==> (parens (generator x (= a as))) emit(ps, mark, K"generator") end end @@ -3071,10 +3075,11 @@ function parse_paren(ps::ParseState, check_identifiers=true) emit(ps, mark, K"block", PARENS_FLAG) else # Parentheses used for grouping - # (a * b) ==> (call-i * a b) - # (a=1) ==> (= a 1) - # (x) ==> x - # (a...) ==> (... a) + # (a * b) ==> (parens (call-i * a b)) + # (a=1) ==> (parens (= a 1)) + # (x) ==> (parens x) + # (a...) ==> (parens (... a)) + emit(ps, mark, K"parens") end end end @@ -3144,8 +3149,8 @@ function parse_brackets(after_parse::Function, continue elseif k == K"for" # Generator syntax - # (x for a in as) ==> (generator x (= a as)) - # (x \n\n for a in as) ==> (generator x (= a as)) + # (x for a in as) ==> (parens (generator x (= a as))) + # (x \n\n for a in as) ==> (parens (generator x (= a as))) parse_generator(ps, mark) else # Error - recovery done when consuming closing_kind @@ -3203,8 +3208,8 @@ function parse_string(ps::ParseState, raw::Bool) bump(ps, TRIVIA_FLAG) k = peek(ps) if k == K"(" - # "a $(x + y) b" ==> (string "a " (call-i x + y) " b") - # "hi$("ho")" ==> (string "hi" (string "ho")) + # "a $(x + y) b" ==> (string "a " (parens (call-i x + y)) " b") + # "hi$("ho")" ==> (string "hi" (parens (string "ho"))) parse_atom(ps) elseif k == K"var" # var identifiers disabled in strings @@ -3346,7 +3351,7 @@ function parse_string(ps::ParseState, raw::Bool) end # String interpolations # "$x$y$z" ==> (string x y z) - # "$(x)" ==> (string x) + # "$(x)" ==> (string (parens x)) # "$x" ==> (string x) # """$x""" ==> (string-s x) # @@ -3440,7 +3445,7 @@ function parse_atom(ps::ParseState, check_identifiers=true) # Being inside quote makes keywords into identifiers at the # first level of nesting # :end ==> (quote end) - # :(end) ==> (quote (error (end))) + # :(end) ==> (quote (parens (error-t))) # Being inside quote makes end non-special again (issue #27690) # a[:(end)] ==> (ref a (quote (error-t end))) parse_atom(ParseState(ps, end_symbol=false), false) diff --git a/test/parser.jl b/test/parser.jl index cc3c02c7..6adddbfa 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -181,9 +181,9 @@ tests = [ JuliaSyntax.parse_juxtapose => [ "2x" => "(juxtapose 2 x)" "2x" => "(juxtapose 2 x)" - "2(x)" => "(juxtapose 2 x)" - "(2)(3)x" => "(juxtapose 2 3 x)" - "(x-1)y" => "(juxtapose (call-i x - 1) y)" + "2(x)" => "(juxtapose 2 (parens x))" + "(2)(3)x" => "(juxtapose (parens 2) (parens 3) x)" + "(x-1)y" => "(juxtapose (parens (call-i x - 1)) y)" "x'y" => "(juxtapose (call-post x ') y)" # errors "\"a\"\"b\"" => "(juxtapose (string \"a\") (error-t) (string \"b\"))" @@ -226,11 +226,14 @@ tests = [ # Prefix function calls for operators which are both binary and unary "+(a,b)" => "(call + a b)" ".+(a,)" => "(call .+ a)" - "(.+)(a)" => "(call (. +) a)" + "(.+)(a)" => "(call (parens (. +)) a)" "+(a=1,)" => "(call + (= a 1))" => Expr(:call, :+, Expr(:kw, :a, 1)) "+(a...)" => "(call + (... a))" "+(a;b,c)" => "(call + a (parameters b c))" "+(;a)" => "(call + (parameters a))" + "+()" => "(call +)" + "+(\n;a)" => "(call + (parameters a))" + "+(\n)" => "(call +)" # Whitespace not allowed before prefix function call bracket "+ (a,b)" => "(call + (error) a b)" # Prefix calls have higher precedence than ^ @@ -238,14 +241,14 @@ tests = [ "+(a,b)(x)^2" => "(call-i (call (call + a b) x) ^ 2)" "<:(a,)" => "(<: a)" # Unary function calls with brackets as grouping, not an arglist - ".+(a)" => "(dotcall-pre + a)" + ".+(a)" => "(dotcall-pre + (parens a))" "+(a;b)" => "(call-pre + (block-p a b))" - "+(a=1)" => "(call-pre + (= a 1))" => Expr(:call, :+, Expr(:(=), :a, 1)) + "+(a=1)" => "(call-pre + (parens (= a 1)))" => Expr(:call, :+, Expr(:(=), :a, 1)) # Unary operators have lower precedence than ^ - "+(a)^2" => "(call-pre + (call-i a ^ 2))" - ".+(a)^2" => "(dotcall-pre + (call-i a ^ 2))" - "+(a)(x,y)^2" => "(call-pre + (call-i (call a x y) ^ 2))" - "<:(a)" => "(<:-pre a)" + "+(a)^2" => "(call-pre + (call-i (parens a) ^ 2))" + ".+(a)^2" => "(dotcall-pre + (call-i (parens a) ^ 2))" + "+(a)(x,y)^2" => "(call-pre + (call-i (call (parens a) x y) ^ 2))" + "<:(a)" => "(<:-pre (parens a))" # Normal unary calls "+x" => "(call-pre + x)" "√x" => "(call-pre √ x)" @@ -276,7 +279,7 @@ tests = [ "<: \n" => "<:" "<: =" => "<:" "<:{T}(x::T)" => "(call (curly <: T) (::-i x T))" - "<:(x::T)" => "(<:-pre (::-i x T))" + "<:(x::T)" => "(<:-pre (parens (::-i x T)))" "<: x" => "(<:-pre x)" "<: <: x" => "(<:-pre (<:-pre x))" "<: A where B" => "(<:-pre (where A B))" @@ -307,11 +310,11 @@ tests = [ "\$A.@x" => "(macrocall (. (\$ A) (quote @x)))" # non-errors in space sensitive contexts - "[f (x)]" => "(hcat f x)" + "[f (x)]" => "(hcat f (parens x))" "[f x]" => "(hcat f x)" # space separated macro calls "@foo a b" => "(macrocall @foo a b)" - "@foo (x)" => "(macrocall @foo x)" + "@foo (x)" => "(macrocall @foo (parens x))" "@foo (x,y)" => "(macrocall @foo (tuple-p x y))" "A.@foo a b" => "(macrocall (. A (quote @foo)) a b)" "@A.foo a b" => "(macrocall (. A (quote @foo)) a b)" @@ -341,7 +344,7 @@ tests = [ Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)) "f(a; b; c)" => "(call f a (parameters b) (parameters c))" => Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a) - "(a=1)()" => "(call (= a 1))" => Expr(:call, Expr(:(=), :a, 1)) + "(a=1)()" => "(call (parens (= a 1)))" => Expr(:call, Expr(:(=), :a, 1)) "f (a)" => "(call f (error-t) a)" "@x(a, b)" => "(macrocall-p @x a b)" "A.@x(y)" => "(macrocall-p (. A (quote @x)) y)" @@ -367,7 +370,10 @@ tests = [ "a[i]" => "(ref a i)" "a [i]" => "(ref a (error-t) i)" "a[i,j]" => "(ref a i j)" - "(a=1)[]" => "(ref (= a 1))" => Expr(:ref, Expr(:(=), :a, 1)) + "(a=1)[]" => "(ref (parens (= a 1)))" => Expr(:ref, Expr(:(=), :a, 1)) + "a[end]" => "(ref a end)" + "a[begin]" => "(ref a begin)" + "a[:(end)]" => "(typed_hcat a (quote (parens (error-t))) (error-t))" "T[x y]" => "(typed_hcat T x y)" "T[x ; y]" => "(typed_vcat T x y)" "T[a b; c d]" => "(typed_vcat T (row a b) (row c d))" @@ -382,13 +388,13 @@ tests = [ "f.(a,b)" => "(dotcall f a b)" "f.(a=1; b=2)" => "(dotcall f (= a 1) (parameters (= b 2)))" => Expr(:., :f, Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))) - "(a=1).()" => "(dotcall (= a 1))" => Expr(:., Expr(:(=), :a, 1), Expr(:tuple)) + "(a=1).()" => "(dotcall (parens (= a 1)))" => Expr(:., Expr(:(=), :a, 1), Expr(:tuple)) "f. (x)" => "(dotcall f (error-t) x)" # Other dotted syntax "A.:+" => "(. A (quote +))" "A.: +" => "(. A (quote (error-t) +))" "f.\$x" => "(. f (inert (\$ x)))" - "f.\$(x+y)" => "(. f (inert (\$ (call-i x + y))))" + "f.\$(x+y)" => "(. f (inert (\$ (parens (call-i x + y)))))" "A.\$B.@x" => "(macrocall (. (. A (inert (\$ B))) (quote @x)))" "@A.\$x a" => "(macrocall (. A (inert (error x))) a)" "A.@x" => "(macrocall (. A (quote @x)))" @@ -496,10 +502,10 @@ tests = [ "export a, \n @b" => "(export a @b)" => Expr(:export, :a, Symbol("@b")) "export +, ==" => "(export + ==)" => Expr(:export, :+, :(==)) "export \n a" => "(export a)" => Expr(:export, :a) - "export \$a, \$(a*b)" => "(export (\$ a) (\$ (call-i a * b)))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b))) - "export (x::T)" => "(export (error (::-i x T)))" + "export \$a, \$(a*b)" => "(export (\$ a) (\$ (parens (call-i a * b))))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b))) + "export (x::T)" => "(export (error (parens (::-i x T))))" "export outer" => "(export outer)" => Expr(:export, :outer) - "export (\$f)" => "(export (\$ f))" => Expr(:export, Expr(:$, :f)) + "export (\$f)" => "(export (parens (\$ f)))" => Expr(:export, Expr(:$, :f)) ], JuliaSyntax.parse_if_elseif => [ "if a xx elseif b yy else zz end" => "(if a (block xx) (elseif b (block yy) (block zz)))" @@ -537,28 +543,27 @@ tests = [ # Macros and functions "macro while(ex) end" => "(macro (call (error while) ex) (block))" "macro f() end" => "(macro (call f) (block))" - "macro (:)(ex) end" => "(macro (call : ex) (block))" - "macro (type)(ex) end" => "(macro (call type ex) (block))" + "macro (:)(ex) end" => "(macro (call (parens :) ex) (block))" + "macro (type)(ex) end" => "(macro (call (parens type) ex) (block))" "macro \$f() end" => "(macro (call (\$ f)) (block))" - "macro (\$f)() end" => "(macro (call (\$ f)) (block))" + "macro (\$f)() end" => "(macro (call (parens (\$ f))) (block))" "function (x) body end"=> "(function (tuple-p x) (block body))" "function (x,y) end" => "(function (tuple-p x y) (block))" "function (x=1) end" => "(function (tuple-p (= x 1)) (block))" "function (;x=1) end" => "(function (tuple-p (parameters (= x 1))) (block))" "function ()(x) end" => "(function (call (tuple-p) x) (block))" - "function (A).f() end" => "(function (call (. A (quote f))) (block))" - "function (:)() end" => "(function (call :) (block))" - "function (x::T)() end"=> "(function (call (::-i x T)) (block))" - "function (::g(x))() end" => "(function (call (::-pre (call g x))) (block))" - "function (f::T{g(i)})() end" => "(function (call (::-i f (curly T (call g i)))) (block))" - "function (::T)() end" => "(function (call (::-pre T)) (block))" + "function (A).f() end" => "(function (call (. (parens A) (quote f))) (block))" + "function (:)() end" => "(function (call (parens :)) (block))" + "function (x::T)() end"=> "(function (call (parens (::-i x T))) (block))" + "function (::g(x))() end" => "(function (call (parens (::-pre (call g x)))) (block))" + "function (f::T{g(i)})() end" => "(function (call (parens (::-i f (curly T (call g i))))) (block))" + "function (::T)() end" => "(function (call (parens (::-pre T))) (block))" "function begin() end" => "(function (call (error begin)) (block))" "function f() end" => "(function (call f) (block))" "function type() end" => "(function (call type) (block))" "function \n f() end" => "(function (call f) (block))" "function \$f() end" => "(function (call (\$ f)) (block))" - "function (:)() end" => "(function (call :) (block))" - "function (::Type{T})(x) end" => "(function (call (::-pre (curly Type T)) x) (block))" + "function (::Type{T})(x) end" => "(function (call (parens (::-pre (curly Type T))) x) (block))" # Function/macro definition with no methods "function f end" => "(function f)" "function f \n\n end" => "(function f)" @@ -576,11 +581,11 @@ tests = [ "function f()::S where T end" => "(function (where (::-i (call f) S) T) (block))" # Ugly cases for compat where extra parentheses existed and we've # already parsed at least the call part of the signature - "function (f() where T) end" => "(function (where (call f) T) (block))" => Expr(:function, Expr(:where, Expr(:call, :f), :T), Expr(:block)) - "function (f()) where T end" => "(function (where (call f) T) (block))" - "function (f() where T) where U end" => "(function (where (where (call f) T) U) (block))" - "function (f()::S) end"=> "(function (::-i (call f) S) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block)) - "function ((f()::S) where T) end" => "(function (where (::-i (call f) S) T) (block))" + "function (f() where T) end" => "(function (parens (where (call f) T)) (block))" => Expr(:function, Expr(:where, Expr(:call, :f), :T), Expr(:block)) + "function (f()) where T end" => "(function (where (parens (call f)) T) (block))" + "function (f() where T) where U end" => "(function (where (parens (where (call f) T)) U) (block))" + "function (f()::S) end"=> "(function (parens (::-i (call f) S)) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block)) + "function ((f()::S) where T) end" => "(function (parens (where (parens (::-i (call f) S)) T)) (block))" # body "function f() \n a \n b end" => "(function (call f) (block a b))" "function f() end" => "(function (call f) (block))" @@ -682,16 +687,16 @@ tests = [ "(a;b;;c)" => "(block-p a b c)" "(a=1; b=2)" => "(block-p (= a 1) (= b 2))" # Parentheses used for grouping - "(a * b)" => "(call-i a * b)" - "(a=1)" => "(= a 1)" - "(x)" => "x" - "(a...)" => "(... a)" + "(a * b)" => "(parens (call-i a * b))" + "(a=1)" => "(parens (= a 1))" + "(x)" => "(parens x)" + "(a...)" => "(parens (... a))" # Generators - "(x for a in as)" => "(generator x (= a as))" - "(x \n\n for a in as)" => "(generator x (= a as))" + "(x for a in as)" => "(parens (generator x (= a as)))" + "(x \n\n for a in as)" => "(parens (generator x (= a as)))" # Range parsing in parens - "(1:\n2)" => "(call-i 1 : 2)" - "(1:2)" => "(call-i 1 : 2)" + "(1:\n2)" => "(parens (call-i 1 : 2))" + "(1:2)" => "(parens (call-i 1 : 2))" ], JuliaSyntax.parse_atom => [ # char literal @@ -741,7 +746,7 @@ tests = [ ":.=" => "(quote .=)" # Special symbols quoted ":end" => "(quote end)" - ":(end)" => "(quote (error-t))" + ":(end)" => "(quote (parens (error-t)))" ":<:" => "(quote <:)" # unexpect = "=" => "(error =)" @@ -759,11 +764,11 @@ tests = [ # parse_generator "[x for a = as for b = bs if cond1 for c = cs if cond2]" => "(comprehension (flatten x (= a as) (filter (= b bs) cond1) (filter (= c cs) cond2)))" "[x for a = as if begin cond2 end]" => "(comprehension (generator x (filter (= a as) (block cond2))))" - "[(x)for x in xs]" => "(comprehension (generator x (error-t) (= x xs)))" - "(a for x in xs if cond)" => "(generator a (filter (= x xs) cond))" - "(xy for x in xs for y in ys)" => "(flatten xy (= x xs) (= y ys))" - "(xy for x in xs for y in ys for z in zs)" => "(flatten xy (= x xs) (= y ys) (= z zs))" - "(x for a in as)" => "(generator x (= a as))" + "[(x)for x in xs]" => "(comprehension (generator (parens x) (error-t) (= x xs)))" + "(a for x in xs if cond)" => "(parens (generator a (filter (= x xs) cond)))" + "(xy for x in xs for y in ys)" => "(parens (flatten xy (= x xs) (= y ys)))" + "(xy for x in xs for y in ys for z in zs)" => "(parens (flatten xy (= x xs) (= y ys) (= z zs)))" + "(x for a in as)" => "(parens (generator x (= a as)))" # parse_vect "[x, y]" => "(vect x y)" "[x, y]" => "(vect x y)" @@ -776,7 +781,7 @@ tests = [ # parse_paren ":(=)" => "(quote =)" ":(::)" => "(quote ::)" - "(function f \n end)" => "(function f)" + "(function f \n end)" => "(parens (function f))" # braces "{x y}" => "(bracescat (row x y))" ((v=v"1.7",), "{x ;;; y}") => "(bracescat (nrow-3 x y))" @@ -845,8 +850,8 @@ tests = [ ((v=v"1.7",), "[;;]") => "(ncat-2 (error))" # parse_string "\"\"\"\n\$x\n a\"\"\"" => "(string-s x \"\\n\" \" a\")" - "\"a \$(x + y) b\"" => "(string \"a \" (call-i x + y) \" b\")" - "\"hi\$(\"ho\")\"" => "(string \"hi\" (string \"ho\"))" + "\"a \$(x + y) b\"" => "(string \"a \" (parens (call-i x + y)) \" b\")" + "\"hi\$(\"ho\")\"" => "(string \"hi\" (parens (string \"ho\")))" "\"a \$foo b\"" => "(string \"a \" foo \" b\")" "\"\$var\"" => "(string var)" "\"\$outer\"" => "(string outer)" @@ -888,7 +893,7 @@ tests = [ "\"str" => "(string \"str\" (error-t))" # String interpolations "\"\$x\$y\$z\"" => "(string x y z)" - "\"\$(x)\"" => "(string x)" + "\"\$(x)\"" => "(string (parens x))" "\"\$x\"" => "(string x)" # Strings with embedded whitespace trivia "\"a\\\nb\"" => raw"""(string "a" "b")""" @@ -931,11 +936,11 @@ end parseall_test_specs = [ # whitespace before keywords in space-insensitive mode - "(y::\nif x z end)" => "(toplevel (::-i y (if x (block z))))" + "(y::\nif x z end)" => "(toplevel (parens (::-i y (if x (block z)))))" # The following may not be ideal error recovery! But at least the parser # shouldn't crash - "@(x y)" => "(toplevel (macrocall x (error-t @y)))" + "@(x y)" => "(toplevel (macrocall (error x (error-t y))))" "|(&\nfunction" => "(toplevel (call | (& (function (error (error)) (block (error)) (error-t))) (error-t)))" ] diff --git a/test/parser_api.jl b/test/parser_api.jl index bc6a1aa8..719af8bb 100644 --- a/test/parser_api.jl +++ b/test/parser_api.jl @@ -3,11 +3,7 @@ @test parse(Expr, " x ") == :x @test JuliaSyntax.remove_linenums!(parseall(Expr, " x ")) == Expr(:toplevel, :x) @test parseatom(Expr, " x ") == :x - # TODO: Fix this situation with trivia here; the brackets are trivia, but - # must be parsed to discover the atom inside. But in GreenTree we only - # place trivia as siblings of the leaf node with identifier `x`, not as - # children. - @test_broken parseatom(Expr, "(x)") == :x + @test parseatom(Expr, "(x)") == :x # SubString @test parse(Expr, SubString("x+y")) == :(x+y)