diff --git a/spec/compiler/crystal/tools/expand_spec.cr b/spec/compiler/crystal/tools/expand_spec.cr new file mode 100644 index 000000000000..cf39abc9a125 --- /dev/null +++ b/spec/compiler/crystal/tools/expand_spec.cr @@ -0,0 +1,474 @@ +require "spec" +require "../../../../src/compiler/crystal/**" + +include Crystal + +private def processed_expand_visitor(code, cursor_location) + compiler = Compiler.new + compiler.no_codegen = true + compiler.no_cleanup = true + compiler.wants_doc = true + result = compiler.compile(Compiler::Source.new(".", code), "fake-no-build") + + visitor = ExpandVisitor.new(cursor_location) + process_result = visitor.process(result) + + {visitor, process_result} +end + +private def run_expand_tool(code) + cursor_location = nil + + code.lines.each_with_index do |line, line_number_0| + if column_number = line.index('‸') + cursor_location = Location.new(".", line_number_0 + 1, column_number + 1) + end + end + + code = code.gsub('‸', "") + + if cursor_location + visitor, result = processed_expand_visitor(code, cursor_location) + + yield result + else + raise "no cursor found in spec" + end +end + +private def expansion_to_a(expansion) + [expansion.original_source].concat(expansion.expanded_sources) +end + +private def assert_expand(code, expected_result) + run_expand_tool code do |result| + result.status.should eq("ok") + result.message.should eq("#{expected_result.size} expansion#{expected_result.size >= 2 ? "s" : ""} found") + result.expansions.not_nil!.zip(expected_result) do |expansion, expected_result| + expansion_to_a(expansion).zip(expected_result) do |result, expected| + result.should eq(expected) + end + end + end +end + +private def assert_expand_simple(code, expanded, original = code.gsub('‸', "")) + assert_expand(code, [[original, expanded]]) +end + +private def assert_expand_fail(code, message = "no expansion found") + run_expand_tool code do |result| + result.status.should eq("failed") + result.message.should eq(message) + end +end + +describe "expand" do + it "expands macro expression {{ ... }}" do + code = "‸{{ 1 + 2 }}" + + assert_expand_simple code, "3" + end + + it "expands macro expression {{ ... }} with cursor inside it" do + code = "{{ 1 ‸+ 2 }}" + + assert_expand_simple code, "3" + end + + it "expands macro expression {{ ... }} with cursor end of it" do + code = "{{ 1 + 2 }‸}" + + assert_expand_simple code, "3" + end + + it "expands macro expression {% ... %}" do + code = %(‸{% "test" %}) + + assert_expand_simple code, "" + end + + it "expands macro expression {% ... %} with cursor at end of it" do + code = %({% "test" ‸%}) + + assert_expand_simple code, "" + end + + it "expands macro control {% if %}" do + code = <<-CODE + {%‸ if 1 == 1 %} + true + {% end %} + CODE + + assert_expand_simple code, "true" + end + + it "expands macro control {% if %} with cursor inside it" do + code = <<-CODE + {% if 1 == 1 %} + tr‸ue + {% end %} + CODE + + assert_expand_simple code, "true" + end + + it "expands macro control {% if %} with cursor at end of it" do + code = <<-CODE + {% if 1 == 1 %} + true + {% end ‸%} + CODE + + assert_expand_simple code, "true" + end + + it "expands macro control {% if %} with indent" do + code = <<-CODE + begin + {% if 1 == 1 %} + t‸rue + {% end %} + end + CODE + + original = <<-CODE + {% if 1 == 1 %} + true + {% end %} + CODE + + assert_expand_simple code, original: original, expanded: "true" + end + + it "expands macro control {% for %}" do + code = <<-CODE + {% f‸or x in 1..3 %} + {{ x }} + {% end %} + CODE + + assert_expand_simple code, "1\n2\n3\n" + end + + it "expands macro control {% for %} with cursor inside it" do + code = <<-CODE + {% for x in 1..3 %} + ‸ {{ x }} + {% end %} + CODE + + assert_expand_simple code, "1\n2\n3\n" + end + + it "expands macro control {% for %} with cursor at end of it" do + code = <<-CODE + {% for x in 1..3 %} + {{ x }} + ‸{% end %} + CODE + + assert_expand_simple code, "1\n2\n3\n" + end + + it "expands macro control {% for %} with indent" do + code = <<-CODE + begin + {% f‸or x in 1..3 %} + {{ x }} + {% end %} + end + CODE + + original = <<-CODE + {% for x in 1..3 %} + {{ x }} + {% end %} + CODE + + assert_expand_simple code, original: original, expanded: "1\n2\n3\n" + end + + it "expands simple macro" do + code = <<-CODE + macro foo + 1 + end + + ‸foo + CODE + + assert_expand_simple code, original: "foo", expanded: "1" + end + + it "expands simple macro with cursor inside it" do + code = <<-CODE + macro foo + 1 + end + + f‸oo + CODE + + assert_expand_simple code, original: "foo", expanded: "1" + end + + it "expands simple macro with cursor at end of it" do + code = <<-CODE + macro foo + 1 + end + + fo‸o + CODE + + assert_expand_simple code, original: "foo", expanded: "1" + end + + it "expands complex macro" do + code = <<-CODE + macro foo + {% if true %} + "if true" + {% end %} + {% for x in %w(1 2 3) %} + {{ x }} + {% end %} + end + + ‸foo + CODE + + assert_expand_simple code, original: "foo", expanded: %("if true"\n"1"\n"2"\n"3"\n) + end + + it "expands macros with 2 level" do + code = <<-CODE + macro foo + :foo + end + + macro bar + foo + :bar + end + + b‸ar + CODE + + assert_expand code, [["bar", "foo\n:bar\n", ":foo\n:bar\n"]] + end + + it "expands macros with 3 level" do + code = <<-CODE + macro foo + :foo + end + + macro bar + foo + :bar + end + + macro baz + foo + bar + :baz + end + + ba‸z + CODE + + assert_expand code, [["baz", "foo\nbar\n:baz\n", ":foo\nfoo\n:bar\n:baz\n", ":foo\n:foo\n:bar\n:baz\n"]] + end + + it "expands macro of module" do + code = <<-CODE + module Foo + macro foo + :Foo + :foo + end + end + + Foo.f‸oo + CODE + + assert_expand_simple code, original: "Foo.foo", expanded: ":Foo\n:foo\n" + end + + it "expands macro of module with cursor at module name" do + code = <<-CODE + module Foo + macro foo + :Foo + :foo + end + end + + F‸oo.foo + CODE + + assert_expand_simple code, original: "Foo.foo", expanded: ":Foo\n:foo\n" + end + + it "expands macro of module with cursor at dot" do + code = <<-CODE + module Foo + macro foo + :Foo + :foo + end + end + + Foo‸.foo + CODE + + assert_expand_simple code, original: "Foo.foo", expanded: ":Foo\n:foo\n" + end + + it "expands macro of module inside module" do + code = <<-CODE + module Foo + macro foo + :Foo + :foo + end + + f‸oo + end + CODE + + assert_expand_simple code, original: "foo", expanded: ":Foo\n:foo\n" + end + + %w(module class struct lib enum).each do |keyword| + it "expands macro expression inside #{keyword}" do + code = <<-CODE + #{keyword} Foo + ‸{{ "Foo = 1".id }} + end + CODE + + assert_expand_simple code, original: %({{ "Foo = 1".id }}), expanded: "Foo = 1" + end + end + + %w(struct union).each do |keyword| + it "expands macro expression inside C #{keyword}" do + code = <<-CODE + lib Foo + #{keyword} Foo + ‸{{ "Foo = 1".id }} + end + end + CODE + + assert_expand_simple code, original: %({{ "Foo = 1".id }}), expanded: "Foo = 1" + end + end + + it "expands macro expression inside def" do + code = <<-CODE + def foo(x : T) forall T + ‸{{ T }} + end + + foo 1 + foo "bar" + CODE + + assert_expand code, [ + ["{{ T }}", "Int32"], + ["{{ T }}", "String"], + ] + end + + it "expands macro expression inside def of module" do + code = <<-CODE + module Foo(T) + def self.foo + {{ ‸T }} + end + end + + Foo(Int32).foo + Foo(String).foo + Foo(1).foo + CODE + + assert_expand code, [ + ["{{ T }}", "Int32"], + ["{{ T }}", "String"], + ["{{ T }}", "1"], + ] + end + + it "doesn't expand macro expression" do + code = <<-CODE + {{ 1 + 2 }} + ‸ + CODE + + assert_expand_fail code + end + + it "doesn't expand macro expression with cursor out of end" do + code = <<-CODE + {{ 1 + 2 }}‸ + CODE + + assert_expand_fail code + end + + it "doesn't expand macro expression" do + code = <<-CODE + ‸ {{ 1 + 2 }} + CODE + + assert_expand_fail code + end + + it "doesn't expand normal call" do + code = <<-CODE + def foo + 1 + end + + ‸foo + CODE + + assert_expand_fail code, "no expansion found: foo is not macro" + end + + it "expands macro with doc" do + code = <<-CODE + macro foo(x) + # string of {{ x }} + def {{ x }}_str + {{ x.stringify }} + end + # symbol of {{ x }} + def {{ x }}_sym + :{{ x }} + end + end + + ‸foo(hello) + CODE + + expanded = <<-CODE + # string of hello + def hello_str + "hello" + end + # symbol of hello + def hello_sym + :hello + end + CODE + + assert_expand_simple code, original: "foo(hello)", expanded: expanded + "\n" + end +end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 770881a9fc22..d446a0ab98b3 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -35,6 +35,7 @@ class Crystal::Command Tool: context show context for given location + expand show macro expansion for given location format format project, directories and/or files hierarchy show type hierarchy implementations show implementations for given call in location @@ -141,6 +142,9 @@ class Crystal::Command when "format".starts_with?(tool) options.shift format + when "expand".starts_with?(tool) + options.shift + expand when "hierarchy".starts_with?(tool) options.shift hierarchy @@ -198,9 +202,10 @@ class Crystal::Command end end - private def compile_no_codegen(command, wants_doc = false, hierarchy = false, cursor_command = false, top_level = false) + private def compile_no_codegen(command, wants_doc = false, hierarchy = false, no_cleanup = false, cursor_command = false, top_level = false) config = create_compiler command, no_codegen: true, hierarchy: hierarchy, cursor_command: cursor_command config.compiler.no_codegen = true + config.compiler.no_cleanup = no_cleanup config.compiler.wants_doc = wants_doc result = top_level ? config.top_level_semantic : config.compile {config, result} diff --git a/src/compiler/crystal/command/cursor.cr b/src/compiler/crystal/command/cursor.cr index b4673b129c55..5a0e7979754d 100644 --- a/src/compiler/crystal/command/cursor.cr +++ b/src/compiler/crystal/command/cursor.cr @@ -17,8 +17,14 @@ class Crystal::Command end end - private def cursor_command(command) - config, result = compile_no_codegen command, cursor_command: true + private def expand + cursor_command("tool expand", no_cleanup: true, wants_doc: true) do |location, config, result| + result = ExpandVisitor.new(location).process(result) + end + end + + private def cursor_command(command, no_cleanup = false, wants_doc = false) + config, result = compile_no_codegen command, cursor_command: true, no_cleanup: no_cleanup, wants_doc: wants_doc format = config.output_format diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 799d0c4f46ee..3f22f504099e 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -60,6 +60,9 @@ module Crystal # If `false`, color won't be used in output messages. property? color = true + # If `true`, skip cleanup process on semantic analysis. + property? no_cleanup = false + # If `true`, no executable will be generated after compilation # (useful to type-check a prorgam) property? no_codegen = false @@ -131,7 +134,7 @@ module Crystal source = [source] unless source.is_a?(Array) program = new_program(source) node = parse program, source - node = program.semantic node, @stats + node = program.semantic node, @stats, cleanup: !no_cleanup? codegen program, node, source, output_filename unless @no_codegen Result.new program, node end diff --git a/src/compiler/crystal/semantic.cr b/src/compiler/crystal/semantic.cr index 2e37b1120498..f1bef5c3f1d9 100644 --- a/src/compiler/crystal/semantic.cr +++ b/src/compiler/crystal/semantic.cr @@ -18,7 +18,7 @@ class Crystal::Program # Runs semantic analysis on the given node, returning a node # that's typed. In the process types and methods are defined in # this program. - def semantic(node : ASTNode, stats = false) : ASTNode + def semantic(node : ASTNode, stats = false, cleanup = true) : ASTNode node, processor = top_level_semantic(node, stats: stats) Crystal.timing("Semantic (cvars initializers)", stats) do @@ -35,15 +35,18 @@ class Crystal::Program end result = Crystal.timing("Semantic (main)", stats) do - visit_main(node, process_finished_hooks: true) + visit_main(node, process_finished_hooks: true, cleanup: cleanup) end + Crystal.timing("Semantic (cleanup)", stats) do cleanup_types cleanup_files end + Crystal.timing("Semantic (recursive struct check)", stats) do RecursiveStructChecker.new(self).run end + result end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index e8c565fbba70..180f9356516f 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -2,7 +2,7 @@ require "./semantic_visitor" module Crystal class Program - def visit_main(node, visitor = MainVisitor.new(self), process_finished_hooks = false) + def visit_main(node, visitor = MainVisitor.new(self), process_finished_hooks = false, cleanup = true) node.accept visitor program.process_finished_hooks(visitor) if process_finished_hooks @@ -10,7 +10,7 @@ module Crystal node.accept missing_types program.process_finished_hooks(missing_types) if process_finished_hooks - node = cleanup node + node = cleanup node if cleanup if process_finished_hooks finished_hooks.map! do |hook| diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 777d27f81d88..f1a425112663 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -2792,10 +2792,12 @@ module Crystal def parse_percent_macro_expression raise "can't nest macro expressions", @token if @in_macro_expression + location = @token.location macro_exp = parse_macro_expression check_macro_expression_end + end_location = token_end_location next_token - MacroExpression.new(macro_exp) + MacroExpression.new(macro_exp).at(location).at_end(end_location) end def parse_macro_expression @@ -2870,7 +2872,7 @@ module Crystal next_token_skip_space check :"%}" - return MacroFor.new(vars, exp, body) + return MacroFor.new(vars, exp, body).at_end(token_end_location) when :if return parse_macro_if(start_line, start_column, macro_state) when :unless @@ -2894,7 +2896,7 @@ module Crystal next_token_skip_space check :"%}" - return MacroIf.new(BoolLiteral.new(true), body) + return MacroIf.new(BoolLiteral.new(true), body).at_end(token_end_location) when :else, :elsif, :end return nil end @@ -2904,7 +2906,7 @@ module Crystal exps = parse_expressions @in_macro_expression = false - MacroExpression.new(exps, output: false) + MacroExpression.new(exps, output: false).at_end(token_end_location) end def parse_macro_if(start_line, start_column, macro_state, check_end = true) @@ -2916,7 +2918,7 @@ module Crystal if @token.type != :"%}" && check_end an_if = parse_if_after_condition cond, true - return MacroExpression.new(an_if, output: false) + return MacroExpression.new(an_if, output: false).at_end(token_end_location) end check :"%}" @@ -2956,7 +2958,7 @@ module Crystal unexpected_token end - return MacroIf.new(cond, a_then, a_else) + return MacroIf.new(cond, a_then, a_else).at_end(token_end_location) end def parse_expression_inside_macro @@ -3664,7 +3666,7 @@ module Crystal end end node.doc = doc - node.location = block.try(&.location) || location + node.location = location node.end_location = block.try(&.end_location) || call_args.try(&.end_location) || end_location node end @@ -4738,6 +4740,7 @@ module Crystal end def parse_lib + location = @token.location next_token_skip_space_or_newline name = check_const @@ -4747,9 +4750,10 @@ module Crystal body = push_visbility { parse_lib_body_expressions } check_ident :end + end_location = token_end_location next_token_skip_space - LibDef.new name, body, name_column_number + LibDef.new(name, body, name_column_number).at(location).at_end(end_location) end def parse_lib_body @@ -4836,6 +4840,7 @@ module Crystal IdentOrConst = [:IDENT, :CONST] def parse_fun_def(require_body = false) + location = @token.location doc = @token.doc push_def if require_body @@ -4914,6 +4919,7 @@ module Crystal if require_body if @token.keyword?(:end) body = Nop.new + end_location = token_end_location next_token else body = parse_expressions @@ -4921,13 +4927,14 @@ module Crystal end else body = nil + end_location = token_end_location end pop_def if require_body fun_def = FunDef.new name, args, return_type, varargs, body, real_name fun_def.doc = doc - fun_def + fun_def.at(location).at_end(end_location) end def parse_alias @@ -5008,14 +5015,16 @@ module Crystal end def parse_c_struct_or_union(union : Bool) + location = @token.location next_token_skip_space_or_newline name = check_const next_token_skip_statement_end body = parse_c_struct_or_union_body_expressions check_ident :end + end_location = token_end_location next_token_skip_space - CStructOrUnionDef.new name, Expressions.from(body), union: union + CStructOrUnionDef.new(name, Expressions.from(body), union: union).at(location).at_end(end_location) end def parse_c_struct_or_union_body @@ -5082,6 +5091,7 @@ module Crystal end def parse_enum_def + location = @token.location doc = @token.doc next_token_skip_space_or_newline @@ -5103,13 +5113,14 @@ module Crystal members = parse_enum_body_expressions check_ident :end + end_location = token_end_location next_token_skip_space raise "Bug: EnumDef name can only be a Path" unless name.is_a?(Path) enum_def = EnumDef.new name, members, base_type enum_def.doc = doc - enum_def + enum_def.at(location).at_end(end_location) end def parse_enum_body diff --git a/src/compiler/crystal/tools/expand.cr b/src/compiler/crystal/tools/expand.cr new file mode 100644 index 000000000000..a29c5977d558 --- /dev/null +++ b/src/compiler/crystal/tools/expand.cr @@ -0,0 +1,190 @@ +module Crystal + struct ExpandResult + JSON.mapping({ + status: {type: String}, + message: {type: String}, + expansions: {type: Array(Expansion), nilable: true}, + }) + + def initialize(@status, @message) + end + + def to_text(io) + io.puts message + expansions.try &.each_with_index do |expansion, i| + io.puts "expansion #{i + 1}:" + io << " " + io.puts expansion.original_source.lines(chomp: false).join " " + io.puts + expansion.expanded_sources.each_with_index do |expanded_source, j| + io << "~> " + io.puts expanded_source.lines(chomp: false).join " " + io.puts + end + end + end + + struct Expansion + JSON.mapping({ + original_source: {type: String}, + expanded_sources: {type: Array(String)}, + }) + + def initialize(@original_source, @expanded_sources) + end + + def self.build(original_node) + transformer = ExpandTransformer.new + expanded_node = transformer.transform original_node + + expanded_sources = [] of String + + while transformer.expanded? + expanded_sources << ast_to_s expanded_node + transformer.expanded = false + expanded_node = transformer.transform expanded_node + end + + Expansion.new ast_to_s(original_node), expanded_sources + end + + private def self.ast_to_s(node) + source = String.build { |io| node.to_s(io, emit_doc: true) } + + # Re-indentation is needed for `MacroIf` and `MacroFor`, because they have + # `MacroBody`, which is sub string of source code, in other words they may + # contain source code's indent. + return source unless node.is_a?(MacroIf) || node.is_a?(MacroFor) + + indent = node.location.not_nil!.column_number - 1 + source.lines(chomp: false).map do |line| + i = 0 + line.each_char do |c| + break unless c.ascii_whitespace? && i < indent + i += 1 + end + line[{i, indent}.min..-1] + end.join + end + end + end + + class ExpandVisitor < Visitor + def initialize(@target_location : Location) + @found_nodes = [] of ASTNode + @in_defs = false + @message = "no expansion found" + end + + def process_type(type) + if type.is_a?(NamedType) + type.types?.try &.values.each do |inner_type| + process_type(inner_type) + end + end + + process_type type.metaclass if type.metaclass != type + + if type.is_a?(DefInstanceContainer) + type.def_instances.values.try do |typed_defs| + typed_defs.each do |typed_def| + typed_def.accept(self) + end + end + end + + if type.is_a?(GenericType) + type.generic_types.values.each do |instanced_type| + process_type(instanced_type) + end + end + end + + def process(result : Compiler::Result) + @in_defs = true + result.program.def_instances.each_value do |typed_def| + typed_def.accept(self) + end + + result.program.types?.try &.values.each do |type| + process_type type + end + @in_defs = false + + result.node.accept(self) + + if @found_nodes.empty? + return ExpandResult.new("failed", @message) + else + res = ExpandResult.new("ok", "#{@found_nodes.size} expansion#{@found_nodes.size > 1 ? "s" : ""} found") + res.expansions = @found_nodes.map { |node| ExpandResult::Expansion.build(node) } + return res + end + end + + def visit(node : Def | FunDef) + @in_defs && contains_target(node) + end + + def visit(node : Call) + if loc_start = node.location + # If node.obj (a.k.a. an receiver) is a Path, it may be macro call and node.obj has no expansion surely. + # Otherwise, we cannot decide node.obj has no expansion. + loc_start = node.name_location unless node.obj.is_a?(Path) + loc_end = node.name_end_location + if @target_location.between?(loc_start, loc_end) + if node.expanded + @found_nodes << node + else + @message = "no expansion found: #{node} is not macro" + end + false + else + contains_target(node) + end + end + end + + def visit(node : MacroFor | MacroIf | MacroExpression) + if loc_start = node.location + loc_end = node.end_location || loc_start + if @target_location.between?(loc_start, loc_end) && node.expanded + @found_nodes << node + false + else + contains_target(node) + end + end + end + + def visit(node) + contains_target(node) + end + + private def contains_target(node) + if loc_start = node.location + loc_end = node.end_location || loc_start + # if it is not between, it could be the case that node is the top level Expressions + # in which the (start) location might be in one file and the end location in another. + @target_location.between?(loc_start, loc_end) || loc_start.filename != loc_end.filename + else + # if node has no location, assume they may contain the target. + # for example with the main expressions ast node this matters + true + end + end + end + + class ExpandTransformer < Transformer + property? expanded = false + + def transform(node : Call | MacroFor | MacroIf | MacroExpression) + if expanded = node.expanded + self.expanded = true + expanded + else + super + end + end + end +end