Fix parser to never create doc from trailing comment (#11268)

spec/compiler/crystal/tools/doc/method_spec.cr

@@ -157,6 +157,29 @@ describe Doc::Method do
       method.doc_copied_from.should be_nil
     end
 
+    it "trailing comment is not a doc comment" do
+      program = semantic(<<-CR, inject_primitives: false, wants_doc: true).program
+        nil # trailing comment
+        def foo
+        end
+        CR
+      generator = Doc::Generator.new program, [""]
+      method = generator.type(program).lookup_class_method("foo").not_nil!
+      method.doc.should be_nil
+    end
+
+    it "trailing comment is not part of a doc comment" do
+      program = semantic(<<-CR, inject_primitives: false, wants_doc: true).program
+        nil # trailing comment
+        # doc comment
+        def foo
+        end
+        CR
+      generator = Doc::Generator.new program, [""]
+      method = generator.type(program).lookup_class_method("foo").not_nil!
+      method.doc.should eq("doc comment")
+    end
+
     it "inherits doc from ancestor (no extra comment)" do
       program = semantic("
         class Foo

src/compiler/crystal/syntax/lexer.cr

@@ -66,6 +66,7 @@ module Crystal
       @filename = ""
       @wants_regex = true
       @doc_enabled = false
+      @comment_is_doc = true
       @comments_enabled = false
       @count_whitespace = false
       @slash_is_regex = true
@@ -96,6 +97,16 @@ module Crystal
     end
 
     def next_token
+      # Check previous token:
+      if @token.type.newline? || @token.type.eof?
+        # 1) After a newline or at the start of the stream (:EOF), a following comment can be a doc comment
+        @comment_is_doc = true
+      elsif !@token.type.space?
+        # 2) Any non-space token prevents a following comment from being a doc
+        # comment.
+        @comment_is_doc = false
+      end
+
       reset_token
 
       # Skip comments
@@ -112,7 +123,7 @@ module Crystal
           consume_loc_pragma
           start = current_pos
         else
-          if @doc_enabled
+          if @doc_enabled && @comment_is_doc
             consume_doc
           elsif @comments_enabled
             return consume_comment(start)