Merge pull request #244 from sebastianpech/disable-unicode-escape

Add argument to weave function to disable unicode escape

src/Weave.jl

@@ -76,14 +76,17 @@ Weave an input document to output file.
 * `cache_path`: where of cached output will be saved.
 * `cache`: controls caching of code: `:off` = no caching, `:all` = cache everything,
   `:user` = cache based on chunk options, `:refresh`, run all code chunks and save new cache.
-* `throw_errors` if `false` errors are included in output document and the whole document is
+* `throw_errors`: if `false` errors are included in output document and the whole document is
     executed. if `true` errors are thrown when they occur.
-* `template` : Template (file path) or MustacheTokens for md2html or md2tex formats.
-* `highlight_theme` : Theme (Highlights.AbstractTheme) for used syntax highlighting
-* `css` : CSS (file path) used for md2html format
-* `pandoc_options` = String array of options to pass to pandoc for `pandoc2html` and
+* `template`: Template (file path) or MustacheTokens for md2html or md2tex formats.
+* `highlight_theme`: Theme (Highlights.AbstractTheme) for used syntax highlighting
+* `css`: CSS (file path) used for md2html format
+* `pandoc_options`: String array of options to pass to pandoc for `pandoc2html` and
    `pandoc2pdf` formats e.g. ["--toc", "-N"]
-* `latex_cmd` the command used to make pdf from .tex
+* `latex_cmd`: the command used to make pdf from .tex
+* `latex_keep_unicode`: if set to true (default is false), do not convert unicode characters to their
+respective latex representation. This is especially useful if a font and tex-engine with support for unicode
+characters are used.
 
 **Note:** Run Weave from terminal and not using IJulia, Juno or ESS, they tend to mess with capturing output.
 """
@@ -95,12 +98,13 @@ function weave(source ; doctype = :auto,
         throw_errors = false,
         template = nothing, highlight_theme = nothing, css = nothing,
         pandoc_options = String[]::Array{String},
-        latex_cmd = "xelatex")
+        latex_cmd = "xelatex",latex_keep_unicode=false)
 
     doc = read_doc(source, informat)
     doctype == :auto && (doctype = detect_doctype(doc.source))
     doc.doctype = doctype
 
+
     # Read args from document header, overrides command line args
     if haskey(doc.header, "options")
         (doctype, informat, out_path, args, mod, fig_path, fig_ext,
@@ -122,7 +126,7 @@ function weave(source ; doctype = :auto,
               mod = mod,
               out_path=out_path, args = args,
               fig_path = fig_path, fig_ext = fig_ext, cache_path = cache_path, cache=cache,
-              throw_errors = throw_errors)
+              throw_errors = throw_errors,latex_keep_unicode=latex_keep_unicode)
       formatted = format(doc)
 
       outname = get_outname(out_path, doc)

src/format.jl

@@ -157,7 +157,8 @@ function format_chunk(chunk::DocChunk, formatdict, docformat::JMarkdown2tex)
         end
     end
     ioformat!(io, out)
-    return uc2tex(String(take!(out)))
+    formatdict[:keep_unicode] || return uc2tex(String(take!(out)))
+    return String(take!(out))
 end
 
 function format_chunk(chunk::DocChunk, formatdict, docformat::JMarkdown2HTML)
@@ -262,7 +263,10 @@ function format_output(result::AbstractString, docformat::JMarkdown2HTML)
 end
 
 function format_output(result::AbstractString, docformat::JMarkdown2tex)
-  return uc2tex(result, true)
+  # Highligts has some extra escaping defined, eg of $, ", ...
+  result_escaped = sprint( (io, x) -> Highlights.Format.escape(io, MIME("text/latex"), x, charescape=true), result)
+  docformat.formatdict[:keep_unicode] || return uc2tex(result_escaped, true)
+  return result_escaped
 end
 
 function format_code(result::AbstractString, docformat)
@@ -272,7 +276,8 @@ end
 function format_code(result::AbstractString, docformat::JMarkdown2tex)
   highlighted = highlight(MIME("text/latex"), strip(result),
         Highlights.Lexers.JuliaLexer, docformat.formatdict[:theme])
-  return uc2tex(highlighted)
+  docformat.formatdict[:keep_unicode] || return uc2tex(highlighted)
+  return highlighted
   #return "\\begin{minted}[mathescape, fontsize=\\small, xleftmargin=0.5em]{julia}\n$result\n\\end{minted}\n"
 end
 

src/formatters.jl

@@ -18,7 +18,8 @@ const tex = Tex("Latex with custom code environments",
                                          :fig_env=> "figure",
                                          :fig_pos => "htpb",
                                          :doctype => "tex",
-                                         :mimetypes => ["application/pdf", "image/png", "text/latex", "text/plain"]
+                                         :mimetypes => ["application/pdf", "image/png", "text/latex", "text/plain"],
+                                         :keep_unicode => false,
                                          ))
 
 const texminted = Tex("Latex using minted for highlighting",
@@ -35,7 +36,8 @@ const texminted = Tex("Latex using minted for highlighting",
                                          :fig_env=> "figure",
                                          :fig_pos => "htpb",
                                          :doctype => "texminted",
-                                         :mimetypes => ["application/pdf", "image/png", "text/latex", "text/plain"]
+                                         :mimetypes => ["application/pdf", "image/png", "text/latex", "text/plain"],
+                                         :keep_unicode => false,
                                          ))
 
 struct Pandoc
@@ -147,7 +149,8 @@ const md2tex = JMarkdown2tex("Julia markdown to latex", Dict{Symbol,Any}(
         :out_width => "\\linewidth",
         :mimetypes => ["application/pdf", "image/png", "image/jpg", "text/latex",
                         "text/markdown", "text/plain"],
-        :doctype=> "md2tex"))
+        :doctype=> "md2tex",
+        :keep_unicode=>false))
 
 
 struct MultiMarkdown

src/run.jl

@@ -26,13 +26,18 @@ Run code chunks and capture output from parsed document.
 function Base.run(doc::WeaveDoc; doctype = :auto,
         mod::Union{Module, Symbol} = :sandbox, out_path=:doc,
         args=Dict(), fig_path = "figures", fig_ext = nothing,
-        cache_path = "cache", cache = :off, throw_errors=false)
+        cache_path = "cache", cache = :off, throw_errors=false, latex_keep_unicode=false)
     #cache :all, :user, :off, :refresh
 
     doc.cwd = get_cwd(doc, out_path)
     doctype == :auto && (doctype = detect_doctype(doc.source))
     doc.doctype = doctype
     doc.format = formats[doctype]
+
+    if (haskey(doc.format.formatdict, :keep_unicode))
+        doc.format.formatdict[:keep_unicode] = latex_keep_unicode
+    end
+
     isdir(doc.cwd) || mkpath(doc.cwd)
 
     if occursin("2pdf", doctype) && cache == :off

test/documents/inline/markdown_beamer.tex.ref

@@ -9,7 +9,7 @@ Some inline output
 
 
 \begin{lstlisting}
-(*@\HLJLnf{println}@*)(*@\HLJLp{(}@*)(*@\HLJLs{"Testing output"}@*)(*@\HLJLp{)}@*)
+(*@\HLJLnf{println}@*)(*@\HLJLp{(}@*)(*@\HLJLs{"{}Testing{\mbox{\space}}output"{}}@*)(*@\HLJLp{)}@*)
 \end{lstlisting}
 
 \begin{lstlisting}

test/formatter_test.jl

@@ -112,3 +112,42 @@ ldoc = Weave.run(parsed, doctype = "md2tex")
 mdoc = Weave.run(parsed, doctype = "github")
 @test mdoc.chunks[1].rich_output == "\n\n### Small markdown sample\n\n**Hello** from `code` block.\n\n"
 @test mdoc.chunks[2].rich_output == "\n\n* one\n* two\n* three\n\n"
+
+
+# Test disable escaping of unicode
+content = """
+# Test chunk
+α
+"""
+
+dchunk = Weave.DocChunk(content, 1, 1)
+
+pformat = Weave.formats["md2tex"]
+
+f = Weave.format_chunk(dchunk, pformat.formatdict, pformat)
+@test f == "\\section{Test chunk}\n\\ensuremath{\\alpha}\n\n"
+pformat.formatdict[:keep_unicode] = true
+f = Weave.format_chunk(dchunk, pformat.formatdict, pformat)
+@test f == "\\section{Test chunk}\nα\n\n"
+
+function doc_from_string(str)
+    parsed = Weave.parse_doc(str,"markdown")
+    header = Weave.parse_header(parsed[1])
+    Weave.WeaveDoc("",parsed,header)
+end
+
+doc_content = """
+```julia
+α = 10
+```
+"""
+
+parsed = doc_from_string(doc_content)
+ldoc = Weave.run(parsed, doctype = "md2tex")
+@test occursin(Weave.uc2tex("α"),Weave.format(ldoc))
+@test !occursin("α",Weave.format(ldoc))
+
+parsed = doc_from_string(doc_content)
+ldoc = Weave.run(parsed, doctype = "md2tex",latex_keep_unicode=true)
+@test occursin("α",Weave.format(ldoc))
+@test !occursin(Weave.uc2tex("α"),Weave.format(ldoc))