diff --git a/src/parser_api.jl b/src/parser_api.jl index bc45d22e..c48fef60 100644 --- a/src/parser_api.jl +++ b/src/parser_api.jl @@ -77,7 +77,7 @@ end function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version=VERSION, ignore_trivia=true, filename=nothing, first_line=1, ignore_errors=false, - ignore_warnings=ignore_errors) where {T} + ignore_warnings=ignore_errors, kws...) where {T} stream = ParseStream(text, index; version=version) if ignore_trivia && rule != :all bump_trivia(stream, skip_newlines=true) @@ -99,7 +99,7 @@ function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version= # * It's kind of required for GreenNode, as GreenNode only records spans, # not absolute positions. # * Dropping it would be ok for SyntaxNode and Expr... - tree = build_tree(T, stream; wrap_toplevel_as_kind=K"toplevel", filename=filename, first_line=first_line) + tree = build_tree(T, stream; wrap_toplevel_as_kind=K"toplevel", filename=filename, first_line=first_line, kws...) tree, last_byte(stream) + 1 end diff --git a/src/syntax_tree.jl b/src/syntax_tree.jl index 7f87a81a..76c4e3d6 100644 --- a/src/syntax_tree.jl +++ b/src/syntax_tree.jl @@ -44,7 +44,13 @@ end Base.show(io::IO, ::ErrorVal) = printstyled(io, "✘", color=:light_red) -function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::Integer=1) +function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}; + keep_parens=false, position::Integer=1) + _to_SyntaxNode(source, raw, convert(Int, position), keep_parens) +end + +function _to_SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, + position::Int, keep_parens::Bool) if !haschildren(raw) && !(is_syntax_kind(raw) || is_keyword(raw)) # Here we parse the values eagerly rather than representing them as # strings. Maybe this is good. Maybe not. @@ -56,10 +62,13 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In for (i,rawchild) in enumerate(children(raw)) # FIXME: Allowing trivia is_error nodes here corrupts the tree layout. if !is_trivia(rawchild) || is_error(rawchild) - push!(cs, SyntaxNode(source, rawchild, pos)) + push!(cs, _to_SyntaxNode(source, rawchild, pos, keep_parens)) end pos += rawchild.span end + if !keep_parens && kind(raw) == K"parens" && length(cs) == 1 + return cs[1] + end node = SyntaxNode(nothing, cs, SyntaxData(source, raw, position, nothing)) for c in cs c.parent = node @@ -182,10 +191,11 @@ end # shallow-copy the data Base.copy(data::SyntaxData) = SyntaxData(data.source, data.raw, data.position, data.val) -function build_tree(::Type{SyntaxNode}, stream::ParseStream; filename=nothing, first_line=1, kws...) +function build_tree(::Type{SyntaxNode}, stream::ParseStream; + filename=nothing, first_line=1, keep_parens=false, kws...) green_tree = build_tree(GreenNode, stream; kws...) source = SourceFile(sourcetext(stream), filename=filename, first_line=first_line) - SyntaxNode(source, green_tree, first_byte(stream)) + SyntaxNode(source, green_tree, position=first_byte(stream), keep_parens=keep_parens) end #------------------------------------------------------------------------------- diff --git a/test/expr.jl b/test/expr.jl index 4cdc23c6..31c47ce9 100644 --- a/test/expr.jl +++ b/test/expr.jl @@ -5,9 +5,9 @@ (s; kws...) -> JuliaSyntax.parsestmt(Expr, s; kws...), (s; kws...) -> JuliaSyntax.parseall(Expr, s; kws...)) else - ((s; kws...) -> Expr(JuliaSyntax.parseatom(SyntaxNode, s; kws...)), - (s; kws...) -> Expr(JuliaSyntax.parsestmt(SyntaxNode, s; kws...)), - (s; kws...) -> Expr(JuliaSyntax.parseall(SyntaxNode, s; kws...))) + ((s; kws...) -> Expr(JuliaSyntax.parseatom(SyntaxNode, s; keep_parens=true, kws...)), + (s; kws...) -> Expr(JuliaSyntax.parsestmt(SyntaxNode, s; keep_parens=true, kws...)), + (s; kws...) -> Expr(JuliaSyntax.parseall(SyntaxNode, s; keep_parens=true, kws...))) end @testset "Quote nodes" begin @@ -534,6 +534,16 @@ @test parsestmt("return") == Expr(:return, nothing) end + @testset "Large integer macros" begin + @test parsestmt("0x00000000000000001") == + Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), + nothing, "0x00000000000000001") + + @test parsestmt("(0x00000000000000001)") == + Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), + nothing, "0x00000000000000001") + end + @testset "struct" begin @test parsestmt("struct A end") == Expr(:struct, false, :A, Expr(:block, LineNumberNode(1))) diff --git a/test/parser.jl b/test/parser.jl index 078c2cb3..e549b668 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -7,7 +7,7 @@ function parse_to_sexpr_str(production, code::AbstractString; v=v"1.6", expr=fal JuliaSyntax.validate_tokens(stream) t = build_tree(GreenNode, stream, wrap_toplevel_as_kind=K"None") source = SourceFile(code) - s = SyntaxNode(source, t) + s = SyntaxNode(source, t, keep_parens=true) if expr JuliaSyntax.remove_linenums!(Expr(s)) else