Fix escaping/interpolating in string and charlist literals in Elixir …

…lexer (#1308)

The Elixir lexer does not properly handle escaping and interpolating in
string (`"foo"`) and charlist (`'foo'`) literals.

This PR adds general escaping rules for use in strings as well as adding
support for escaping and interpolating to charlist literals. It also
rationalises the states used for string literals. Examples to confirm
correct behaviour have been added to the visual sample.

lib/rouge/lexers/elixir.rb

@@ -51,15 +51,25 @@ class Elixir < RegexLexer
       state :strings do
         rule %r/(%[A-Ba-z])?"""(?:.|\n)*?"""/, Str::Doc
         rule %r/'''(?:.|\n)*?'''/, Str::Doc
-        rule %r/"/, Str::Doc, :dqs
-        rule %r/'.*?'/, Str::Single
+        rule %r/"/, Str::Double, :dqs
+        rule %r/'/, Str::Single, :sqs
         rule %r{(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b[^x0MC])|(\\[MC]-)+\w|[^\s\\])}, Str::Other
-
       end
 
       state :dqs do
+        mixin :escapes
+        mixin :interpoling
+        rule %r/[^#"\\]+/, Str::Double
         rule %r/"/, Str::Double, :pop!
-        mixin :enddoublestr
+        rule %r/[#\\]/, Str::Double
+      end
+
+      state :sqs do
+        mixin :escapes
+        mixin :interpoling
+        rule %r/[^#'\\]+/, Str::Single
+        rule %r/'/, Str::Single, :pop!
+        rule %r/[#\\]/, Str::Single
       end
 
       state :interpoling do
@@ -71,17 +81,18 @@ class Elixir < RegexLexer
         mixin :root
       end
 
+      state :escapes do
+        rule %r/\\x\h{2}/, Str::Escape
+        rule %r/\\u\{?\d+\}?/, Str::Escape
+        rule %r/\\[\\abdefnrstv0"']/, Str::Escape
+      end
+
       state :interpoling_symbol do
         rule %r/"/, Str::Symbol, :pop!
         mixin :interpoling
         rule %r/[^#"]+/, Str::Symbol
       end
 
-      state :enddoublestr do
-        mixin :interpoling
-        rule %r/[^#"]+/, Str::Double
-      end
-
       state :sigil_strings do
         # ~-sigiled strings
         # ~(abc), ~[abc], ~<abc>, ~|abc|, ~r/abc/, etc

spec/visual/samples/elixir

@@ -20,6 +20,20 @@
 # maps
 %{key: 1}
 
+# strings
+"Hello world"
+"Hello \" world"
+"Hello \n world"
+"Hello #{"world"}"
+"Hello #world"
+
+# charlists
+'Hello world'
+'Hello \' world'
+'Hello \n world'
+'Hello #{'world'}'
+'Hello #world'
+
 # regexes
 ~r()gif
 ~r[]gfi