Use flags for struct/module variants and allow empty return nodes

Various AST cleanups:

* `struct` and `mutable struct` are distinguished with flags not children

* `module` and `baremodule` are distinguished with flags not children

* `return` now parses with empty children rather than with an invisible
  K"nothing" as the only child.

src/expr.jl

@@ -278,6 +278,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
             pushfirst!(args[2].args, loc)
         end
     elseif headsym === :module
+        pushfirst!(args, !has_flags(node, BARE_MODULE_FLAG))
         pushfirst!(args[3].args, loc)
     elseif headsym == :inert || (headsym == :quote && length(args) == 1 &&
                  !(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
@@ -303,11 +304,13 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
             args[1] = Expr(headsym, args[1].args...)
             headsym = :const
         end
-    elseif headsym == :return && isempty(args)
+    elseif headsym === :return && isempty(args)
         push!(args, nothing)
-    elseif headsym == :juxtapose
+    elseif headsym === :juxtapose
         headsym = :call
         pushfirst!(args, :*)
+    elseif headsym === :struct
+        pushfirst!(args, has_flags(node, MUTABLE_FLAG))
     end
     return Expr(headsym, args...)
 end

src/hooks.jl

@@ -62,7 +62,7 @@ function _incomplete_tag(n::SyntaxNode)
     elseif kp in KSet"for while function if"
         return i == 1 ? :other : :block
     elseif kp in KSet"module struct"
-        return i == 2 ? :other : :block
+        return i == 1 ? :other : :block
     elseif kp == K"do"
         return i < 3  ? :other : :block
     else

src/parse_stream.jl

@@ -5,27 +5,38 @@
 # TODO: Use `primitive type SyntaxFlags 16 end` rather than an alias?
 const RawFlags = UInt16
 const EMPTY_FLAGS = RawFlags(0)
-# Applied to tokens which are syntax trivia after parsing
+
+# Set for tokens or ranges which are syntax trivia after parsing
 const TRIVIA_FLAG = RawFlags(1<<0)
 
-# Record whether operators are dotted
+# Token flags - may be set for operator kinded tokens
+# Operator is dotted
 const DOTOP_FLAG = RawFlags(1<<1)
-# Record whether operator has a suffix
-const SUFFIXED_FLAG        = RawFlags(1<<2)
+# Operator has a suffix
+const SUFFIXED_FLAG = RawFlags(1<<2)
 
+# Set for K"call", K"dotcall" or any syntactic operator heads
 # Distinguish various syntaxes which are mapped to K"call"
 const PREFIX_CALL_FLAG = RawFlags(0<<3)
 const INFIX_FLAG       = RawFlags(1<<3)
 const PREFIX_OP_FLAG   = RawFlags(2<<3)
 const POSTFIX_OP_FLAG  = RawFlags(3<<3)
 
-# The next two bits could overlap with the previous two if necessary
-# Set when kind == K"String" was triple-delimited as with """ or ```
+# The following flags are quite head-specific and may overlap
+
+# Set when K"string" or K"cmdstring" was triple-delimited as with """ or ```
 const TRIPLE_STRING_FLAG = RawFlags(1<<5)
-# Set when a string or identifier needs "raw string" unescaping
+# Set when a K"string", K"cmdstring" or K"Identifier" needs raw string unescaping
 const RAW_STRING_FLAG = RawFlags(1<<6)
 
-const PARENS_FLAG = RawFlags(1<<7)
+# Set for K"tuple", K"block" or K"macrocall" which are delimited by parentheses
+const PARENS_FLAG = RawFlags(1<<5)
+
+# Set for K"struct" when mutable
+const MUTABLE_FLAG = RawFlags(1<<5)
+
+# Set for K"module" when it's not bare (`module`, not `baremodule`)
+const BARE_MODULE_FLAG = RawFlags(1<<5)
 
 # Flags holding the dimension of an nrow or other UInt8 not held in the source
 const NUMERIC_FLAGS = RawFlags(RawFlags(0xff)<<8)
@@ -78,10 +89,18 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
         is_infix_op_call(head)   && (str = str*"-i")
         is_prefix_op_call(head)  && (str = str*"-pre")
         is_postfix_op_call(head) && (str = str*"-post")
-        has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s")
-        has_flags(head, RAW_STRING_FLAG) && (str = str*"-r")
-        has_flags(head, PARENS_FLAG) && (str = str*"-p")
-        is_suffixed(head) && (str = str*"-S")
+
+        if kind(head) in KSet"string cmdstring Identifier"
+            has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s")
+            has_flags(head, RAW_STRING_FLAG) && (str = str*"-r")
+        elseif kind(head) in KSet"tuple block macrocall"
+            has_flags(head, PARENS_FLAG) && (str = str*"-p")
+        elseif kind(head) == K"struct"
+            has_flags(head, MUTABLE_FLAG) && (str = str*"-mut")
+        elseif kind(head) == K"module"
+            has_flags(head, BARE_MODULE_FLAG) && (str = str*"-bare")
+        end
+        is_suffixed(head) && (str = str*"-suf")
         n = numeric_flags(head)
         n != 0 && (str = str*"-"*string(n))
     end

src/parser.jl

@@ -763,7 +763,7 @@ function parse_comparison(ps::ParseState, subtype_comparison=false)
     mark = position(ps)
     if subtype_comparison && is_reserved_word(peek(ps))
         # Recovery
-        # struct try end  ==>  (struct false (error (try)) (block))
+        # struct try end  ==>  (struct (error (try)) (block))
         name = untokenize(peek(ps))
         bump(ps)
         emit(ps, mark, K"error", error="Invalid type name `$name`")
@@ -1764,8 +1764,8 @@ function parse_struct_field(ps::ParseState)
     parse_eq(ps)
     if const_field
         # Const fields https://github.com/JuliaLang/julia/pull/43305
-        #v1.8: struct A const a end  ==>  (struct false A (block (const x)))
-        #v1.7: struct A const a end  ==>  (struct false A (block (error (const x))))
+        #v1.8: struct A const a end  ==>  (struct A (block (const x)))
+        #v1.7: struct A const a end  ==>  (struct A (block (error (const x))))
         emit(ps, mark, K"const")
         min_supported_version(v"1.8", ps, mark, "`const` struct field")
     end
@@ -1929,24 +1929,23 @@ function parse_resword(ps::ParseState)
         bump_closing_token(ps, K"end")
         emit(ps, mark, K"abstract")
     elseif word in KSet"struct mutable"
-        # struct A <: B \n a::X \n end  ==>  (struct false (<: A B) (block (::-i a X)))
-        # struct A \n a \n b \n end  ==>  (struct false A (block a b))
-        #v1.7: struct A const a end  ==>  (struct false A (block (error (const a))))
-        #v1.8: struct A const a end  ==>  (struct false A (block (const a)))
-        if word == K"mutable"
-            # mutable struct A end  ==>  (struct true A (block))
+        # struct A <: B \n a::X \n end  ==>  (struct (<: A B) (block (::-i a X)))
+        # struct A \n a \n b \n end  ==>  (struct A (block a b))
+        #v1.7: struct A const a end  ==>  (struct A (block (error (const a))))
+        #v1.8: struct A const a end  ==>  (struct A (block (const a)))
+        is_mut = word == K"mutable"
+        if is_mut
+            # mutable struct A end  ==>  (struct-mut A (block))
             bump(ps, TRIVIA_FLAG)
-            bump_invisible(ps, K"true")
         else
-            # struct A end  ==>  (struct false A (block))
-            bump_invisible(ps, K"false")
+            # struct A end  ==>  (struct A (block))
         end
         @check peek(ps) == K"struct"
         bump(ps, TRIVIA_FLAG)
         parse_subtype_spec(ps)
         parse_block(ps, parse_struct_field)
         bump_closing_token(ps, K"end")
-        emit(ps, mark, K"struct")
+        emit(ps, mark, K"struct", is_mut ? MUTABLE_FLAG : EMPTY_FLAGS)
     elseif word == K"primitive"
         # primitive type A 32 end             ==> (primitive A 32)
         # primitive type A 32 ; end           ==> (primitive A 32)
@@ -1986,22 +1985,22 @@ function parse_resword(ps::ParseState)
                     error="unexpected token after $(untokenize(word))")
         end
     elseif word in KSet"module baremodule"
-        # module A end  ==> (module true A (block))
-        # baremodule A end ==> (module false A (block))
+        # module A end  ==> (module A (block))
+        # baremodule A end ==> (module-bare A (block))
         bump(ps, TRIVIA_FLAG)
-        bump_invisible(ps, (word == K"module") ? K"true" : K"false")
         if is_reserved_word(peek(ps))
-            # module do \n end  ==>  (module true (error do) (block))
+            # module do \n end  ==>  (module (error do) (block))
             bump(ps, error="Invalid module name")
         else
-            # module $A end  ==>  (module true ($ A) (block))
+            # module $A end  ==>  (module ($ A) (block))
             parse_unary_prefix(ps)
         end
-        # module A \n a \n b \n end  ==>  (module true A (block a b))
-        # module A \n "x"\na \n end  ==>  (module true A (block (doc (string "x") a)))
+        # module A \n a \n b \n end  ==>  (module A (block a b))
+        # module A \n "x"\na \n end  ==>  (module A (block (doc (string "x") a)))
         parse_block(ps, parse_docstring)
         bump_closing_token(ps, K"end")
-        emit(ps, mark, K"module")
+        emit(ps, mark, K"module",
+             word == K"baremodule" ? BARE_MODULE_FLAG : EMPTY_FLAGS)
     elseif word == K"export"
         # export a         ==>  (export a)
         # export @a        ==>  (export @a)

test/expr.jl

@@ -312,4 +312,18 @@
         @test parse(Expr, "return x") == Expr(:return, :x)
         @test parse(Expr, "return")  == Expr(:return, nothing)
     end
+
+    @testset "struct" begin
+        @test parse(Expr, "struct A end") ==
+            Expr(:struct, false, :A, Expr(:block, LineNumberNode(1)))
+        @test parse(Expr, "mutable struct A end") ==
+            Expr(:struct, true, :A, Expr(:block, LineNumberNode(1)))
+    end
+
+    @testset "module" begin
+        @test parse(Expr, "module A end") ==
+            Expr(:module, true,  :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
+        @test parse(Expr, "baremodule A end") ==
+            Expr(:module, false, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
+    end
 end

test/hooks.jl

@@ -38,8 +38,8 @@
 
     @testset "Expr(:incomplete)" begin
         JuliaSyntax.enable_in_core!()
-
         @test Meta.isexpr(Meta.parse("[x"), :incomplete)
+        JuliaSyntax.enable_in_core!(false)
 
         for (str, tag) in [
                 ""             => :none
@@ -94,7 +94,6 @@
                 @test Base.incomplete_tag(Meta.parse(str, raise=false)) == tag
             end
         end
-        JuliaSyntax.enable_in_core!(false)
 
         # Should not throw
         @test JuliaSyntax._core_parser_hook("+=", "somefile", 1, 0, :statement)[1] isa Expr

test/parser.jl

@@ -467,13 +467,13 @@ tests = [
         "primitive type A \$N end"  =>  "(primitive A (\$ N))"
         "primitive type A <: B \n 8 \n end"  =>  "(primitive (<: A B) 8)"
         # struct
-        "struct A <: B \n a::X \n end" =>  "(struct false (<: A B) (block (::-i a X)))" => Expr(:struct, false, Expr(:<:, :A, :B), Expr(:block, Expr(:(::), :a, :X)))
-        "struct A \n a \n b \n end"    =>  "(struct false A (block a b))"             => Expr(:struct, false, :A, Expr(:block, :a, :b))
-        "mutable struct A end"         =>  "(struct true A (block))"
-        ((v=v"1.8",), "struct A const a end") => "(struct false A (block (const a)))" => Expr(:struct, false, :A, Expr(:block, Expr(:const, :a)))
-        ((v=v"1.7",), "struct A const a end") => "(struct false A (block (error (const a))))"
-        "struct A end"    =>  "(struct false A (block))"  => Expr(:struct, false, :A, Expr(:block))
-        "struct try end"  =>  "(struct false (error (try)) (block))"
+        "struct A <: B \n a::X \n end" =>  "(struct (<: A B) (block (::-i a X)))" => Expr(:struct, false, Expr(:<:, :A, :B), Expr(:block, Expr(:(::), :a, :X)))
+        "struct A \n a \n b \n end"    =>  "(struct A (block a b))"             => Expr(:struct, false, :A, Expr(:block, :a, :b))
+        "mutable struct A end"         =>  "(struct-mut A (block))"
+        ((v=v"1.8",), "struct A const a end") => "(struct A (block (const a)))" => Expr(:struct, false, :A, Expr(:block, Expr(:const, :a)))
+        ((v=v"1.7",), "struct A const a end") => "(struct A (block (error (const a))))"
+        "struct A end"    =>  "(struct A (block))"  => Expr(:struct, false, :A, Expr(:block))
+        "struct try end"  =>  "(struct (error (try)) (block))"
         # return
         "return\nx"   =>  "(return)"
         "return)"     =>  "(return)"
@@ -483,12 +483,12 @@ tests = [
         "break"    => "(break)"
         "continue" => "(continue)"
         # module/baremodule
-        "module A end"      =>  "(module true A (block))"
-        "baremodule A end"  =>  "(module false A (block))"
-        "module do \n end"  =>  "(module true (error (do)) (block))"
-        "module \$A end"    =>  "(module true (\$ A) (block))"
-        "module A \n a \n b \n end"  =>  "(module true A (block a b))"
-        """module A \n "x"\na\n end""" => """(module true A (block (doc (string "x") a)))"""
+        "module A end"      =>  "(module A (block))"
+        "baremodule A end"  =>  "(module-bare A (block))"
+        "module do \n end"  =>  "(module (error (do)) (block))"
+        "module \$A end"    =>  "(module (\$ A) (block))"
+        "module A \n a \n b \n end"  =>  "(module A (block a b))"
+        """module A \n "x"\na\n end""" => """(module A (block (doc (string "x") a)))"""
         # export
         "export a"   =>  "(export a)"  => Expr(:export, :a)
         "export @a"  =>  "(export @a)" => Expr(:export, Symbol("@a"))