Implement non-standard MIME handling

src/server.jl

@@ -48,7 +48,7 @@ function init!(file::File)
 end
 
 function refresh!(file::File, frontmatter::Dict)
-    exeflags = frontmatter["julia"]["exeflags"]
+    exeflags = frontmatter["format"]["metadata"]["julia"]["exeflags"]
     if exeflags != file.exeflags
         Malt.stop(file.worker)
         file.worker = cd(() -> Malt.Worker(; exeflags), dirname(file.path))
@@ -79,10 +79,7 @@ function evaluate!(
     path = abspath(f.path)
     if isfile(path)
         raw_chunks, file_frontmatter = raw_text_chunks(f)
-        # TODO: retrieve `pandoc` options out of here as well so that we can adjust
-        # the output mimetypes in the worker process based on the format.
-        merged_frontmatter = get(options, "metadata", file_frontmatter)
-        merged_frontmatter = _recursive_merge(default_frontmatter(), merged_frontmatter)
+        merged_frontmatter = _extract_relevant_frontmatter(file_frontmatter, options)
         cells = evaluate_raw_cells!(f, raw_chunks, merged_frontmatter; showprogress)
         data = (
             metadata = (
@@ -108,6 +105,77 @@ function evaluate!(
     end
 end
 
+function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict)
+    file_frontmatter = _recursive_merge(default_frontmatter(), file_frontmatter)
+
+    fig_width_default = get(file_frontmatter, "fig-width", nothing)
+    fig_height_default = get(file_frontmatter, "fig-height", nothing)
+    fig_format_default = get(file_frontmatter, "fig-format", nothing)
+    fig_dpi_default = get(file_frontmatter, "fig-dpi", nothing)
+
+    pandoc_to_default = nothing
+
+    julia_default = get(file_frontmatter, "julia", nothing)
+
+    if isempty(options)
+        return _frontmatter_template(;
+            fig_width = fig_width_default,
+            fig_height = fig_height_default,
+            fig_format = fig_format_default,
+            fig_dpi = fig_dpi_default,
+            pandoc_to = pandoc_to_default,
+            julia = julia_default,
+        )
+    else
+        D = Dict{String,Any}
+
+        format = get(D, options, "format")
+        execute = get(D, format, "execute")
+        fig_width = get(execute, "fig-width", fig_width_default)
+        fig_height = get(execute, "fig-height", fig_height_default)
+        fig_format = get(execute, "fig-format", fig_format_default)
+        fig_dpi = get(execute, "fig-dpi", fig_dpi_default)
+
+        pandoc = get(D, format, "pandoc")
+        pandoc_to = get(pandoc, "to", pandoc_to_default)
+
+        metadata = get(D, format, "metadata")
+        julia = get(metadata, "julia", julia_default)
+
+        return _frontmatter_template(;
+            fig_width,
+            fig_height,
+            fig_format,
+            fig_dpi,
+            pandoc_to,
+            julia,
+        )
+    end
+end
+
+function _frontmatter_template(;
+    fig_width,
+    fig_height,
+    fig_format,
+    fig_dpi,
+    pandoc_to,
+    julia,
+)
+    D = Dict{String,Any}
+    return D(
+        "format" => D(
+            "execute" => D(
+                "fig-width" => fig_width,
+                "fig-height" => fig_height,
+                "fig-format" => fig_format,
+                "fig-dpi" => fig_dpi,
+            ),
+            "pandoc" => D("to" => pandoc_to),
+            "metadata" => D("julia" => julia),
+        ),
+    )
+end
+
 function _parsed_options(options::String)
     isfile(options) || error("`options` is not a valid file: $(repr(options))")
     open(options) do io
@@ -192,7 +260,10 @@ function raw_markdown_chunks(path::String)
                         "Cannot handle an `eval` code cell option with value $(repr(evaluate)), only true or false.",
                     )
                 end
-                push!(raw_chunks, (type = :code, source, file = path, line, evaluate))
+                push!(
+                    raw_chunks,
+                    (type = :code, source, file = path, line, evaluate, cell_options),
+                )
             end
         end
         if terminal_line <= length(source_lines)
@@ -300,7 +371,14 @@ function raw_script_chunks(path::String)
                 end
                 push!(
                     raw_chunks,
-                    (type = :code, source, file = path, line = start_line, evaluate),
+                    (;
+                        type = :code,
+                        source,
+                        file = path,
+                        line = start_line,
+                        evaluate,
+                        cell_options,
+                    ),
                 )
             elseif type == :markdown
                 try
@@ -407,7 +485,12 @@ function evaluate_raw_cells!(
             if chunk.evaluate
                 # Offset the line number by 1 to account for the triple backticks
                 # that are part of the markdown syntax for code blocks.
-                expr = :(render($chunk.source, $(chunk.file), $(chunk.line + 1)))
+                expr = :(render(
+                    $chunk.source,
+                    $(chunk.file),
+                    $(chunk.line + 1),
+                    $(chunk.cell_options),
+                ))
                 remote = Malt.remote_eval_fetch(f.worker, expr)
                 processed = process_results(remote.results)
 
@@ -546,6 +629,7 @@ function process_results(dict::Dict{String,@NamedTuple{error::Bool, data::Vector
     funcs = Dict(
         "application/json" => json_reader,
         "text/plain" => String,
+        "text/markdown" => String,
         "text/html" => String,
         "text/latex" => String,
         "image/svg+xml" => String,

src/worker.jl

@@ -81,10 +81,15 @@ function worker_init(f::File)
             return nothing
         end
 
-        function render(code::AbstractString, file::AbstractString, line::Integer)
+        function render(
+            code::AbstractString,
+            file::AbstractString,
+            line::Integer,
+            cell_options::AbstractDict,
+        )
             captured =
                 Base.@invokelatest include_str(WORKSPACE[], code; file = file, line = line)
-            results = Base.@invokelatest render_mimetypes(captured.value)
+            results = Base.@invokelatest render_mimetypes(captured.value, cell_options)
             return (;
                 results,
                 output = captured.output,
@@ -174,7 +179,23 @@ function worker_init(f::File)
         end
 
         # passing our module removes Main.Notebook noise when printing types etc.
-        with_context(io::IO) = IOContext(io, :module => WORKSPACE[], :color => true)
+        function with_context(io::IO, cell_options = Dict{String,Any}())
+            return IOContext(
+                io,
+                :module => WORKSPACE[],
+                :color => true,
+                # This allows a `show` method implementation to check for
+                # metadata that may be of relevance to it's rendering. For
+                # example, if a `typst` table is rendered with a caption
+                # (available in the `cell_options`) then we need to adjust the
+                # syntax that is output via the `QuartoNotebookRunner/typst`
+                # show method to switch between `markdown` and `code` "mode".
+                #
+                # TODO: perhaps preprocess the metadata provided here rather
+                # than just passing it through as-is.
+                :QuartoNotebookRunner => (; cell_options, frontmatter = FRONTMATTER[]),
+            )
+        end
 
         function clean_bt_str(is_error::Bool, bt, err, prefix = "", mimetype = false)
             is_error || return UInt8[]
@@ -200,21 +221,61 @@ function worker_init(f::File)
             return take!(buf)
         end
 
-        function render_mimetypes(value)
+        function render_mimetypes(value, cell_options)
+            to_format = FRONTMATTER[]["format"]["pandoc"]["to"]
+
             result = Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}()
-            mimes = [
-                "text/plain",
-                "text/html",
-                "text/latex",
-                "image/svg+xml",
-                "image/png",
-                "application/json",
-            ]
+            # Some output formats that we want to write to need different
+            # handling of valid MIME types. Currently `docx` and `typst`. When
+            # we detect that the `to` format is one of these then we select a
+            # different set of MIME types to try and render the value as. The
+            # `QuartoNotebookRunner/*` MIME types are unique to this package and
+            # are how package authors can hook into the display system used here
+            # to allow their types to be rendered correctly in different
+            # outputs.
+            #
+            # NOTE: We may revise this approach at any point in time and these
+            # should be considered implementation details until officially
+            # documented.
+            mime_groups = Dict(
+                "docx" => [
+                    "QuartoNotebookRunner/openxml",
+                    "text/plain",
+                    "text/markdown",
+                    "text/latex",
+                    "text/html",
+                    "image/svg+xml",
+                    "image/png",
+                ],
+                "typst" => [
+                    "QuartoNotebookRunner/typst",
+                    "text/plain",
+                    "text/markdown",
+                    "text/latex",
+                    "image/svg+xml",
+                    "image/png",
+                ],
+            )
+            mimes = get(mime_groups, to_format) do
+                [
+                    "text/plain",
+                    "text/markdown",
+                    "text/html",
+                    "text/latex",
+                    "image/svg+xml",
+                    "image/png",
+                    "application/json",
+                ]
+            end
             for mime in mimes
                 if showable(mime, value)
                     buffer = IOBuffer()
                     try
-                        Base.@invokelatest show(with_context(buffer), mime, value)
+                        Base.@invokelatest show(
+                            with_context(buffer, cell_options),
+                            mime,
+                            value,
+                        )
                     catch error
                         backtrace = catch_backtrace()
                         result[mime] = (;
@@ -229,20 +290,46 @@ function worker_init(f::File)
                         )
                         continue
                     end
+                    # See whether the current MIME type needs to be handled
+                    # specially and embedded in a raw markdown block and whether
+                    # we should skip attempting to render any other MIME types
+                    # to may match.
+                    skip_other_mimes, new_mime, new_buffer = _transform_output(mime, buffer)
                     # Only send back the bytes, we do the processing of the
                     # data on the parent process where we have access to
                     # whatever packages we need, e.g. working out the size
                     # of a PNG image or converting a JSON string to an
                     # actual JSON object that avoids double serializing it
                     # in the notebook output.
-                    result[mime] = (; error = false, data = take!(buffer))
+                    result[new_mime] = (; error = false, data = take!(new_buffer))
+                    skip_other_mimes && break
                 end
             end
             return result
         end
-        render_mimetypes(value::Nothing) =
+        render_mimetypes(value::Nothing, cell_options) =
             Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}()
 
+        # Our custom MIME types need special handling. They get rendered to
+        # `text/markdown` blocks with the original content wrapped in a raw
+        # markdown block. MIMEs that don't match just get passed through.
+        function _transform_output(mime::String, buffer::IO)
+            mapping = Dict(
+                "QuartoNotebookRunner/openxml" => (true, "text/markdown", "openxml"),
+                "QuartoNotebookRunner/typst" => (true, "text/markdown", "typst"),
+            )
+            if haskey(mapping, mime)
+                (skip_other_mimes, mime, raw) = mapping[mime]
+                io = IOBuffer()
+                println(io, "```{=$raw}")
+                println(io, rstrip(read(seekstart(buffer), String)))
+                println(io, "```")
+                return (skip_other_mimes, mime, io)
+            else
+                return (false, mime, buffer)
+            end
+        end
+
         # Integrations:
 
         function ojs_define(; kwargs...)
@@ -275,10 +362,10 @@ function worker_init(f::File)
         function _frontmatter()
             fm = FRONTMATTER[]
 
-            fig_width_inch = get(fm, "fig-width", nothing)
-            fig_height_inch = get(fm, "fig-height", nothing)
-            fig_format = fm["fig-format"]
-            fig_dpi = get(fm, "fig-dpi", nothing)
+            fig_width_inch = fm["format"]["execute"]["fig-width"]
+            fig_height_inch = fm["format"]["execute"]["fig-height"]
+            fig_format = fm["format"]["execute"]["fig-format"]
+            fig_dpi = fm["format"]["execute"]["fig-dpi"]
 
             if fig_format == "retina"
                 fig_format = "svg"

test/runtests.jl

@@ -880,7 +880,11 @@ end
                 cp(joinpath(@__DIR__, "examples/mimetypes"), joinpath(dir, "mimetypes"))
                 server = QuartoNotebookRunner.Server()
                 write("mimetypes.qmd", content)
-                options = Dict{String,Any}("metadata" => Dict{String,Any}("fig-dpi" => 100))
+                options = Dict{String,Any}(
+                    "format" => Dict{String,Any}(
+                        "execute" => Dict{String,Any}("fig-dpi" => 100),
+                    ),
+                )
                 json = QuartoNotebookRunner.run!(
                     server,
                     "mimetypes.qmd";