Merge pull request #189 from yui-knk/location

Refactoring Location

Steepfile

@@ -9,12 +9,13 @@ target :lib do
   check "lib/lrama/grammar/percent_code.rb"
   # TODO: Include this file once Lrama::Grammar::Symbol type is defined
   # check "lib/lrama/grammar/reference.rb"
-  check "lib/lrama/lexer/token.rb"
   check "lib/lrama/lexer/token/char.rb"
   check "lib/lrama/lexer/token/ident.rb"
   check "lib/lrama/lexer/token/parameterizing.rb"
   check "lib/lrama/lexer/token/tag.rb"
   check "lib/lrama/lexer/token/user_code.rb"
+  check "lib/lrama/lexer/location.rb"
+  check "lib/lrama/lexer/token.rb"
   check "lib/lrama/report/duration.rb"
   check "lib/lrama/report/profile.rb"
   check "lib/lrama/warning.rb"

lib/lrama/lexer.rb

@@ -1,4 +1,5 @@
 require "strscan"
+require "lrama/lexer/location"
 require "lrama/lexer/token"
 
 module Lrama
@@ -31,8 +32,8 @@ class Lexer
 
     def initialize(text)
       @scanner = StringScanner.new(text)
-      @head = @scanner.pos
-      @line = 1
+      @head_column = @head = @scanner.pos
+      @head_line = @line = 1
       @status = :initial
       @end_symbol = nil
     end
@@ -54,6 +55,13 @@ def column
       @scanner.pos - @head
     end
 
+    def location
+      Location.new(
+        first_line: @head_line, first_column: @head_column,
+        last_line: @line, last_column: column
+      )
+    end
+
     def lex_token
       while !@scanner.eos? do
         case
@@ -84,17 +92,17 @@ def lex_token
       when @scanner.scan(/[\?\+\*]/)
         return [@scanner.matched, @scanner.matched]
       when @scanner.scan(/<\w+>/)
-        return [:TAG, setup_token(Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched))]
+        return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)]
       when @scanner.scan(/'.'/)
-        return [:CHARACTER, setup_token(Lrama::Lexer::Token::Char.new(s_value: @scanner.matched))]
+        return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)]
       when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/)
-        return [:CHARACTER, setup_token(Lrama::Lexer::Token::Char.new(s_value: @scanner.matched))]
+        return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)]
       when @scanner.scan(/"/)
         return [:STRING, %Q("#{@scanner.scan_until(/"/)})]
       when @scanner.scan(/\d+/)
         return [:INTEGER, Integer(@scanner.matched)]
       when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/)
-        token = setup_token(Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched))
+        token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location)
         type =
           if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/)
             :IDENT_COLON
@@ -118,13 +126,13 @@ def lex_c_code
         when @scanner.scan(/}/)
           if nested == 0 && @end_symbol == '}'
             @scanner.unscan
-            return [:C_DECLARATION, setup_token(Lrama::Lexer::Token::UserCode.new(s_value: code))]
+            return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)]
           else
             code += @scanner.matched
             nested -= 1
           end
         when @scanner.check(/#{@end_symbol}/)
-          return [:C_DECLARATION, setup_token(Lrama::Lexer::Token::UserCode.new(s_value: code))]
+          return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)]
         when @scanner.scan(/\n/)
           code += @scanner.matched
           newline
@@ -158,13 +166,6 @@ def lex_comment
       end
     end
 
-    def setup_token(token)
-      token.line = @head_line
-      token.column = @head_column
-
-      token
-    end
-
     def newline
       @line += 1
       @head = @scanner.pos + 1

lib/lrama/lexer/location.rb

@@ -0,0 +1,22 @@
+module Lrama
+  class Lexer
+    class Location
+      attr_reader :first_line, :first_column, :last_line, :last_column
+
+      def initialize(first_line:, first_column:, last_line:, last_column:)
+        @first_line = first_line
+        @first_column = first_column
+        @last_line = last_line
+        @last_column = last_column
+      end
+
+      def ==(other)
+        self.class == other.class &&
+        self.first_line == other.first_line &&
+        self.first_column == other.first_column &&
+        self.last_line == other.last_line &&
+        self.last_column == other.last_column
+      end
+    end
+  end
+end

lib/lrama/lexer/token.rb

@@ -1,8 +1,8 @@
 module Lrama
   class Lexer
-    class Token < Struct.new(:s_value, :alias_name, keyword_init: true)
+    class Token < Struct.new(:s_value, :alias_name, :location, keyword_init: true)
 
-      attr_accessor :line, :column, :referred
+      attr_accessor :referred
 
       def to_s
         "#{super} line: #{line}, column: #{column}"
@@ -15,6 +15,24 @@ def referred_by?(string)
       def ==(other)
         self.class == other.class && self.s_value == other.s_value
       end
+
+      def first_line
+        location.first_line
+      end
+      alias :line :first_line
+
+      def first_column
+        location.first_column
+      end
+      alias :column :first_column
+
+      def last_line
+        location.last_line
+      end
+
+      def last_column
+        location.last_column
+      end
     end
   end
 end

lib/lrama/lexer/token/user_code.rb

@@ -4,7 +4,7 @@ class Token
       class UserCode < Token
         attr_accessor :references
 
-        def initialize(s_value: nil, alias_name: nil)
+        def initialize(s_value: nil, alias_name: nil, location: nil)
           super
           self.references = []
         end

parser.y

@@ -409,18 +409,22 @@ def next_token
 end
 
 def on_error(error_token_id, error_value, value_stack)
-  if error_value.respond_to?(:line) && error_value.respond_to?(:column)
-    line = error_value.line
-    first_column = error_value.column
+  if error_value.is_a?(Lrama::Lexer::Token)
+    line = error_value.first_line
+    first_column = error_value.first_column
+    last_column = error_value.last_column
+    value = "'#{error_value.s_value}'"
   else
     line = @lexer.line
     first_column = @lexer.head_column
+    last_column = @lexer.column
+    value = error_value.inspect
   end
 
   raise ParseError, <<~ERROR
-    #{@path}:#{line}:#{first_column}: parse error on value #{error_value.inspect} (#{token_to_str(error_token_id) || '?'})
+    #{@path}:#{line}:#{first_column}: parse error on value #{value} (#{token_to_str(error_token_id) || '?'})
     #{@text.split("\n")[line - 1]}
-    #{carrets(first_column)}
+    #{carrets(first_column, last_column)}
   ERROR
 end
 
@@ -441,6 +445,6 @@ def end_c_declaration
   @lexer.end_symbol = nil
 end
 
-def carrets(first_column)
-  ' ' * (first_column + 1) + '^' * (@lexer.column - first_column)
+def carrets(first_column, last_column)
+  ' ' * (first_column + 1) + '^' * (last_column - first_column)
 end

sig/lrama/lexer/location.rbs

@@ -0,0 +1,14 @@
+module Lrama
+  class Lexer
+    class Location
+      attr_reader first_line: Integer
+      attr_reader first_column: Integer
+      attr_reader last_line: Integer
+      attr_reader last_column: Integer
+
+      def initialize: (first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void
+
+      def ==: (Location other) -> bool
+    end
+  end
+end

sig/lrama/lexer/token.rbs

@@ -3,15 +3,20 @@ module Lrama
     class Token
       attr_accessor s_value: String
       attr_accessor alias_name: String
-      attr_accessor line: Integer
-      attr_accessor column: Integer
+      attr_accessor location: Location
       attr_accessor referred: bool
 
-      def initialize: (?s_value: String, ?alias_name: String) -> void
+      def initialize: (?s_value: String, ?alias_name: String, ?location: Location) -> void
 
       def to_s: () -> String
       def referred_by?: (String string) -> bool
       def ==: (Token other) -> bool
+      def first_line: () -> Integer
+      def first_column: () -> Integer
+      def last_line: () -> Integer
+      def last_column: () -> Integer
+      alias line first_line
+      alias column first_column
     end
   end
 end

spec/lrama/parser_spec.rb

@@ -1512,7 +1512,7 @@ class : keyword_class tSTRING keyword_end { code 1 }
        ;
           INPUT
           expect { Lrama::Parser.new(y, "error_messages/parse.y").parse }.to raise_error(ParseError, <<~ERROR)
-            error_messages/parse.y:5:7: parse error on value #<struct Lrama::Lexer::Token::Ident s_value="invalid", alias_name=nil> (IDENTIFIER)
+            error_messages/parse.y:5:7: parse error on value 'invalid' (IDENTIFIER)
             %expect invalid
                     ^^^^^^^
           ERROR