Improve string literals handling

lib/ripper_ruby_parser/sexp_handlers/string_literals.rb

@@ -123,8 +123,7 @@ def process_symbols(exp)
 
       def process_at_tstring_content(exp)
         _, content, pos, delim = exp.shift 4
-        string = handle_string_unescaping(content, delim)
-        string = handle_string_encoding(string, delim)
+        string = fix_encoding handle_string_unescaping(content, delim)
         with_position(pos, s(:str, string))
       end
 
@@ -252,15 +251,6 @@ def handle_string_unescaping(content, delim)
           content
         end
       end
-
-      def handle_string_encoding(string, delim)
-        case delim
-        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
-          fix_encoding string
-        else
-          string
-        end
-      end
     end
   end
 end

test/ripper_ruby_parser/sexp_handlers/string_literals_test.rb

@@ -206,6 +206,23 @@
           .must_be_parsed_as s(:str, "bar\rbaz\r\n")
       end
 
+      describe "when an encoding comment is used" do
+        it "creates UTF-8 strings regardless" do
+          _("# encoding: ascii-8bit\n\"\\0\"")
+            .must_be_parsed_as s(:str, "\u0000")
+        end
+
+        it "uses UTF8 if multi-byte escapes are used" do
+          _("# encoding: ascii-8bit\n\"\\u00a4\"")
+            .must_be_parsed_as s(:str, "\u00a4")
+        end
+
+        it "keeps unicode encoding for escape multi-byte characters" do
+          _("# encoding: ascii-8bit\n'\\あ'")
+            .must_be_parsed_as s(:str, "\\あ")
+        end
+      end
+
       describe "with double-quoted strings with escape sequences" do
         it "works for strings with escape sequences" do
           _('"\\n"')
@@ -295,6 +312,10 @@
           _('"2\302\275"').must_be_parsed_as s(:str, "2½")
         end
 
+        it "converts hex escapes to unicode if possible" do
+          _('"\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"').must_be_parsed_as s(:str, "日本語")
+        end
+
         it "does not convert to unicode if result is not valid" do
           _('"2\x82\302\275"')
             .must_be_parsed_as s(:str,

test/samples/ascii.rb

@@ -2,3 +2,8 @@
 # frozen_string_literal: true
 
 %Q[foo\n\0\nbar]
+"\u00a4"
+"あ"
+"\あ"
+%q{\あ}
+'\あ'