From 353aeba225cd59711771d2c9fb73ee4e9bca7b89 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Wed, 31 Jan 2024 19:42:54 +0000 Subject: [PATCH 1/9] Implement non-standard MIME handling --- src/server.jl | 22 +++++++-- src/worker.jl | 126 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 130 insertions(+), 18 deletions(-) diff --git a/src/server.jl b/src/server.jl index 42d10c8..db41161 100644 --- a/src/server.jl +++ b/src/server.jl @@ -192,7 +192,10 @@ function raw_markdown_chunks(path::String) "Cannot handle an `eval` code cell option with value $(repr(evaluate)), only true or false.", ) end - push!(raw_chunks, (type = :code, source, file = path, line, evaluate)) + push!( + raw_chunks, + (type = :code, source, file = path, line, evaluate, cell_options), + ) end end if terminal_line <= length(source_lines) @@ -300,7 +303,14 @@ function raw_script_chunks(path::String) end push!( raw_chunks, - (type = :code, source, file = path, line = start_line, evaluate), + (; + type = :code, + source, + file = path, + line = start_line, + evaluate, + cell_options, + ), ) elseif type == :markdown try @@ -407,7 +417,12 @@ function evaluate_raw_cells!( if chunk.evaluate # Offset the line number by 1 to account for the triple backticks # that are part of the markdown syntax for code blocks. - expr = :(render($chunk.source, $(chunk.file), $(chunk.line + 1))) + expr = :(render( + $chunk.source, + $(chunk.file), + $(chunk.line + 1), + $(chunk.cell_options), + )) remote = Malt.remote_eval_fetch(f.worker, expr) processed = process_results(remote.results) @@ -546,6 +561,7 @@ function process_results(dict::Dict{String,@NamedTuple{error::Bool, data::Vector funcs = Dict( "application/json" => json_reader, "text/plain" => String, + "text/markdown" => String, "text/html" => String, "text/latex" => String, "image/svg+xml" => String, diff --git a/src/worker.jl b/src/worker.jl index 64a399d..e72d21d 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -81,10 +81,15 @@ function worker_init(f::File) return nothing end - function render(code::AbstractString, file::AbstractString, line::Integer) + function render( + code::AbstractString, + file::AbstractString, + line::Integer, + cell_options::AbstractDict, + ) captured = Base.@invokelatest include_str(WORKSPACE[], code; file = file, line = line) - results = Base.@invokelatest render_mimetypes(captured.value) + results = Base.@invokelatest render_mimetypes(captured.value, cell_options) return (; results, output = captured.output, @@ -174,7 +179,23 @@ function worker_init(f::File) end # passing our module removes Main.Notebook noise when printing types etc. - with_context(io::IO) = IOContext(io, :module => WORKSPACE[], :color => true) + function with_context(io::IO, cell_options = Dict{String,Any}()) + return IOContext( + io, + :module => WORKSPACE[], + :color => true, + # This allows a `show` method implementation to check for + # metadata that may be of relevance to it's rendering. For + # example, if a `typst` table is rendered with a caption + # (available in the `cell_options`) then we need to adjust the + # syntax that is output via the `QuartoNotebookRunner/typst` + # show method to switch between `markdown` and `code` "mode". + # + # TODO: perhaps preprocess the metadata provided here rather + # than just passing it through as-is. + :QuartoNotebookRunner => (; cell_options, frontmatter = FRONTMATTER[]), + ) + end function clean_bt_str(is_error::Bool, bt, err, prefix = "", mimetype = false) is_error || return UInt8[] @@ -200,21 +221,70 @@ function worker_init(f::File) return take!(buf) end - function render_mimetypes(value) + # TODO: where is this key? + function _to_format() + fm = FRONTMATTER[] + if haskey(fm, "pandoc") + pandoc = fm["pandoc"] + if isa(pandoc, Dict) && haskey(pandoc, "to") + to = pandoc["to"] + isa(to, String) && return to + end + end + return nothing + end + + function render_mimetypes(value, cell_options) + to_format = _to_format() result = Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}() - mimes = [ - "text/plain", - "text/html", - "text/latex", - "image/svg+xml", - "image/png", - "application/json", - ] + # Some output formats that we want to write to need different + # handling of valid MIME types. Currently `docx` and `typst`. When + # we detect that the `to` format is one of these then we select a + # different set of MIME types to try and render the value as. The + # `QuartoNotebookRunner/*` MIME types are unique to this package and + # are how package authors can hook into the display system used here + # to allow their types to be rendered correctly in different + # outputs. + # + # NOTE: We may revise this approach at any point in time and these + # should be considered implementation details until officially + # documented. + mime_groups = Dict( + "docx" => [ + "QuartoNotebookRunner/openxml", + "text/plain", + "text/latex", + "text/html", + "image/svg+xml", + "image/png", + ], + "typst" => [ + "QuartoNotebookRunner/typst", + "text/plain", + "text/latex", + "image/svg+xml", + "image/png", + ], + ) + mimes = get(mime_groups, to_format) do + [ + "text/plain", + "text/html", + "text/latex", + "image/svg+xml", + "image/png", + "application/json", + ] + end for mime in mimes if showable(mime, value) buffer = IOBuffer() try - Base.@invokelatest show(with_context(buffer), mime, value) + Base.@invokelatest show( + with_context(buffer, cell_options), + mime, + value, + ) catch error backtrace = catch_backtrace() result[mime] = (; @@ -229,20 +299,46 @@ function worker_init(f::File) ) continue end + # See whether the current MIME type needs to be handled + # specially and embedded in a raw markdown block and whether + # we should skip attempting to render any other MIME types + # to may match. + skip_other_mimes, new_mime, new_buffer = _transform_output(mime, buffer) # Only send back the bytes, we do the processing of the # data on the parent process where we have access to # whatever packages we need, e.g. working out the size # of a PNG image or converting a JSON string to an # actual JSON object that avoids double serializing it # in the notebook output. - result[mime] = (; error = false, data = take!(buffer)) + result[new_mime] = (; error = false, data = take!(new_buffer)) + skip_other_mimes && break end end return result end - render_mimetypes(value::Nothing) = + render_mimetypes(value::Nothing, cell_options) = Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}() + # Our custom MIME types need special handling. They get rendered to + # `text/markdown` blocks with the original content wrapped in a raw + # markdown block. MIMEs that don't match just get passed through. + function _transform_output(mime::String, buffer::IO) + mapping = Dict( + "QuartoNotebookRunner/openxml" => (true, "text/markdown", "openxml"), + "QuartoNotebookRunner/typst" => (true, "text/markdown", "typst"), + ) + if haskey(mapping, mime) + (skip_other_mimes, mime, raw) = mapping[mime] + io = IOBuffer() + println(io, "```{=$raw}") + println(io, rstrip(read(seekstart(buffer), String))) + println(io, "```") + return (skip_other_mimes, mime, io) + else + return (false, mime, buffer) + end + end + # Integrations: function ojs_define(; kwargs...) From 301e5e001ed4dca3ffbe333ff52339fdc2b17807 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 08:18:30 +0000 Subject: [PATCH 2/9] Include `text/markdown` MIME type in all stacks --- src/worker.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/worker.jl b/src/worker.jl index e72d21d..89ab41b 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -253,6 +253,7 @@ function worker_init(f::File) "docx" => [ "QuartoNotebookRunner/openxml", "text/plain", + "text/markdown", "text/latex", "text/html", "image/svg+xml", @@ -261,6 +262,7 @@ function worker_init(f::File) "typst" => [ "QuartoNotebookRunner/typst", "text/plain", + "text/markdown", "text/latex", "image/svg+xml", "image/png", @@ -269,6 +271,7 @@ function worker_init(f::File) mimes = get(mime_groups, to_format) do [ "text/plain", + "text/markdown", "text/html", "text/latex", "image/svg+xml", From e256e4f4fe5e98392ddce4ab3e400a7f53500748 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 11:21:38 +0000 Subject: [PATCH 3/9] Filter metadata passed to worker processes --- src/server.jl | 78 ++++++++++++++++++++++++++++++++++++++++++++---- src/worker.jl | 24 ++++----------- test/runtests.jl | 6 +++- 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/src/server.jl b/src/server.jl index db41161..74e6358 100644 --- a/src/server.jl +++ b/src/server.jl @@ -48,7 +48,7 @@ function init!(file::File) end function refresh!(file::File, frontmatter::Dict) - exeflags = frontmatter["julia"]["exeflags"] + exeflags = frontmatter["format"]["metadata"]["julia"]["exeflags"] if exeflags != file.exeflags Malt.stop(file.worker) file.worker = cd(() -> Malt.Worker(; exeflags), dirname(file.path)) @@ -79,10 +79,7 @@ function evaluate!( path = abspath(f.path) if isfile(path) raw_chunks, file_frontmatter = raw_text_chunks(f) - # TODO: retrieve `pandoc` options out of here as well so that we can adjust - # the output mimetypes in the worker process based on the format. - merged_frontmatter = get(options, "metadata", file_frontmatter) - merged_frontmatter = _recursive_merge(default_frontmatter(), merged_frontmatter) + merged_frontmatter = _extract_relevant_frontmatter(file_frontmatter, options) cells = evaluate_raw_cells!(f, raw_chunks, merged_frontmatter; showprogress) data = ( metadata = ( @@ -108,6 +105,77 @@ function evaluate!( end end +function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict) + file_frontmatter = _recursive_merge(default_frontmatter(), file_frontmatter) + + fig_width_default = get(file_frontmatter, "fig-width", nothing) + fig_height_default = get(file_frontmatter, "fig-height", nothing) + fig_format_default = get(file_frontmatter, "fig-format", nothing) + fig_dpi_default = get(file_frontmatter, "fig-dpi", nothing) + + pandoc_to_default = nothing + + julia_default = get(file_frontmatter, "julia", nothing) + + if isempty(options) + return _frontmatter_template(; + fig_width = fig_width_default, + fig_height = fig_height_default, + fig_format = fig_format_default, + fig_dpi = fig_dpi_default, + pandoc_to = pandoc_to_default, + julia = julia_default, + ) + else + D = Dict{String,Any} + + format = get(D, options, "format") + execute = get(D, format, "execute") + fig_width = get(execute, "fig-width", fig_width_default) + fig_height = get(execute, "fig-height", fig_height_default) + fig_format = get(execute, "fig-format", fig_format_default) + fig_dpi = get(execute, "fig-dpi", fig_dpi_default) + + pandoc = get(D, format, "pandoc") + pandoc_to = get(pandoc, "to", pandoc_to_default) + + metadata = get(D, format, "metadata") + julia = get(metadata, "julia", julia_default) + + return _frontmatter_template(; + fig_width, + fig_height, + fig_format, + fig_dpi, + pandoc_to, + julia, + ) + end +end + +function _frontmatter_template(; + fig_width, + fig_height, + fig_format, + fig_dpi, + pandoc_to, + julia, +) + D = Dict{String,Any} + return D( + "format" => D( + "execute" => D( + "fig-width" => fig_width, + "fig-height" => fig_height, + "fig-format" => fig_format, + "fig-dpi" => fig_dpi, + ), + "pandoc" => D("to" => pandoc_to), + "metadata" => D("julia" => julia), + ), + ) +end + function _parsed_options(options::String) isfile(options) || error("`options` is not a valid file: $(repr(options))") open(options) do io diff --git a/src/worker.jl b/src/worker.jl index 89ab41b..d54820c 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -221,21 +221,9 @@ function worker_init(f::File) return take!(buf) end - # TODO: where is this key? - function _to_format() - fm = FRONTMATTER[] - if haskey(fm, "pandoc") - pandoc = fm["pandoc"] - if isa(pandoc, Dict) && haskey(pandoc, "to") - to = pandoc["to"] - isa(to, String) && return to - end - end - return nothing - end - function render_mimetypes(value, cell_options) - to_format = _to_format() + to_format = FRONTMATTER[]["format"]["pandoc"]["to"] + result = Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}() # Some output formats that we want to write to need different # handling of valid MIME types. Currently `docx` and `typst`. When @@ -374,10 +362,10 @@ function worker_init(f::File) function _frontmatter() fm = FRONTMATTER[] - fig_width_inch = get(fm, "fig-width", nothing) - fig_height_inch = get(fm, "fig-height", nothing) - fig_format = fm["fig-format"] - fig_dpi = get(fm, "fig-dpi", nothing) + fig_width_inch = fm["format"]["execute"]["fig-width"] + fig_height_inch = fm["format"]["execute"]["fig-height"] + fig_format = fm["format"]["execute"]["fig-format"] + fig_dpi = fm["format"]["execute"]["fig-dpi"] if fig_format == "retina" fig_format = "svg" diff --git a/test/runtests.jl b/test/runtests.jl index d6ad4db..1478f51 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -880,7 +880,11 @@ end cp(joinpath(@__DIR__, "examples/mimetypes"), joinpath(dir, "mimetypes")) server = QuartoNotebookRunner.Server() write("mimetypes.qmd", content) - options = Dict{String,Any}("metadata" => Dict{String,Any}("fig-dpi" => 100)) + options = Dict{String,Any}( + "format" => Dict{String,Any}( + "execute" => Dict{String,Any}("fig-dpi" => 100), + ), + ) json = QuartoNotebookRunner.run!( server, "mimetypes.qmd"; From 07b1d05916f058f0744963dd408f7959f8fcdc89 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 12:10:40 +0000 Subject: [PATCH 4/9] Rename `frontmatter` to `options` where appropriate --- src/server.jl | 34 +++++++++++----------------------- src/worker.jl | 34 +++++++++++++++++----------------- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/src/server.jl b/src/server.jl index 74e6358..066572f 100644 --- a/src/server.jl +++ b/src/server.jl @@ -47,15 +47,15 @@ function init!(file::File) Malt.remote_eval_fetch(worker, worker_init(file)) end -function refresh!(file::File, frontmatter::Dict) - exeflags = frontmatter["format"]["metadata"]["julia"]["exeflags"] +function refresh!(file::File, options::Dict) + exeflags = options["format"]["metadata"]["julia"]["exeflags"] if exeflags != file.exeflags Malt.stop(file.worker) file.worker = cd(() -> Malt.Worker(; exeflags), dirname(file.path)) file.exeflags = exeflags init!(file) end - expr = :(refresh!($(frontmatter))) + expr = :(refresh!($(options))) Malt.remote_eval_fetch(file.worker, expr) end @@ -79,8 +79,8 @@ function evaluate!( path = abspath(f.path) if isfile(path) raw_chunks, file_frontmatter = raw_text_chunks(f) - merged_frontmatter = _extract_relevant_frontmatter(file_frontmatter, options) - cells = evaluate_raw_cells!(f, raw_chunks, merged_frontmatter; showprogress) + merged_options = _extract_relevant_options(file_frontmatter, options) + cells = evaluate_raw_cells!(f, raw_chunks, merged_options; showprogress) data = ( metadata = ( kernelspec = ( @@ -105,7 +105,7 @@ function evaluate!( end end -function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict) +function _extract_relevant_options(file_frontmatter::Dict, options::Dict) file_frontmatter = _recursive_merge(default_frontmatter(), file_frontmatter) fig_width_default = get(file_frontmatter, "fig-width", nothing) @@ -118,7 +118,7 @@ function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict) julia_default = get(file_frontmatter, "julia", nothing) if isempty(options) - return _frontmatter_template(; + return _options_template(; fig_width = fig_width_default, fig_height = fig_height_default, fig_format = fig_format_default, @@ -142,7 +142,7 @@ function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict) metadata = get(D, format, "metadata") julia = get(metadata, "julia", julia_default) - return _frontmatter_template(; + return _options_template(; fig_width, fig_height, fig_format, @@ -153,14 +153,7 @@ function _extract_relevant_frontmatter(file_frontmatter::Dict, options::Dict) end end -function _frontmatter_template(; - fig_width, - fig_height, - fig_format, - fig_dpi, - pandoc_to, - julia, -) +function _options_template(; fig_width, fig_height, fig_format, fig_dpi, pandoc_to, julia) D = Dict{String,Any} return D( "format" => D( @@ -468,13 +461,8 @@ end Evaluate the raw cells in `chunks` and return a vector of cells with the results in all available mimetypes. """ -function evaluate_raw_cells!( - f::File, - chunks::Vector, - frontmatter::Dict; - showprogress = true, -) - refresh!(f, frontmatter) +function evaluate_raw_cells!(f::File, chunks::Vector, options::Dict; showprogress = true) + refresh!(f, options) cells = [] @maybe_progress showprogress "Running $(relpath(f.path, pwd()))" for (nth, chunk) in enumerate(chunks) diff --git a/src/worker.jl b/src/worker.jl index d54820c..b19347b 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -22,11 +22,11 @@ function worker_init(f::File) const PROJECT = Base.active_project() const WORKSPACE = Ref(Module(:Notebook)) - const FRONTMATTER = Ref($(default_frontmatter())) + const OPTIONS = Ref(Dict{String,Any}()) # Interface: - function refresh!(frontmatter = FRONTMATTER[]) + function refresh!(options = OPTIONS[]) # Current directory should always start out as the directory of the # notebook file, which is not necessarily right initially if the parent # process was started from a different directory to the notebook. @@ -66,16 +66,16 @@ function worker_init(f::File) # can immediately activate a project environment if they want to. Core.eval(WORKSPACE[], :(import Main: Pkg, ojs_define)) - # Rerun the package loading hooks if the frontmatter has changed. - if FRONTMATTER[] != frontmatter - FRONTMATTER[] = frontmatter + # Rerun the package loading hooks if the options have changed. + if OPTIONS[] != options + OPTIONS[] = options for (pkgid, hook) in PACKAGE_LOADING_HOOKS if haskey(Base.loaded_modules, pkgid) hook() end end else - FRONTMATTER[] = frontmatter + OPTIONS[] = options end return nothing @@ -193,7 +193,7 @@ function worker_init(f::File) # # TODO: perhaps preprocess the metadata provided here rather # than just passing it through as-is. - :QuartoNotebookRunner => (; cell_options, frontmatter = FRONTMATTER[]), + :QuartoNotebookRunner => (; cell_options, options = OPTIONS[]), ) end @@ -222,7 +222,7 @@ function worker_init(f::File) end function render_mimetypes(value, cell_options) - to_format = FRONTMATTER[]["format"]["pandoc"]["to"] + to_format = OPTIONS[]["format"]["pandoc"]["to"] result = Dict{String,@NamedTuple{error::Bool, data::Vector{UInt8}}}() # Some output formats that we want to write to need different @@ -359,13 +359,13 @@ function worker_init(f::File) end end - function _frontmatter() - fm = FRONTMATTER[] + function _figure_metadata() + options = OPTIONS[] - fig_width_inch = fm["format"]["execute"]["fig-width"] - fig_height_inch = fm["format"]["execute"]["fig-height"] - fig_format = fm["format"]["execute"]["fig-format"] - fig_dpi = fm["format"]["execute"]["fig-dpi"] + fig_width_inch = options["format"]["execute"]["fig-width"] + fig_height_inch = options["format"]["execute"]["fig-height"] + fig_format = options["format"]["execute"]["fig-format"] + fig_dpi = options["format"]["execute"]["fig-dpi"] if fig_format == "retina" fig_format = "svg" @@ -392,7 +392,7 @@ function worker_init(f::File) end function _CairoMakie_hook(pkgid::Base.PkgId, CairoMakie::Module) - fm = _frontmatter() + fm = _figure_metadata() if fm.fig_dpi !== nothing kwargs = Dict{Symbol,Any}( :px_per_unit => fm.fig_dpi / 96, @@ -410,7 +410,7 @@ function worker_init(f::File) _CairoMakie_hook(::Any...) = nothing function _Makie_hook(pkgid::Base.PkgId, Makie::Module) - fm = _frontmatter() + fm = _figure_metadata() # only change Makie theme if sizes are set, if only one is set, pick an aspect ratio of 4/3 # which might be more user-friendly than throwing an error if fm.fig_width_inch !== nothing || fm.fig_height_inch !== nothing @@ -436,7 +436,7 @@ function worker_init(f::File) _Makie_hook(::Any...) = nothing function _Plots_hook(pkgid::Base.PkgId, Plots::Module) - fm = _frontmatter() + fm = _figure_metadata() # Convert inches to CSS pixels or device-independent pixels. # Empirically, an SVG is saved by Plots with width and height taken directly as CSS pixels (without unit specified) # so the conversion with the 96 factor would be correct in that setting. From e1cd7dd42bbe9880a11f0fc21923664f23b7cf7c Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 14:30:26 +0000 Subject: [PATCH 5/9] Tests for typst and docx --- .gitignore | 2 + src/worker.jl | 1 - test/examples/docx_mimetypes.qmd | 394 ++++++++++++++++++++++++++++++ test/examples/typst_mimetypes.qmd | 87 +++++++ test/runtests.jl | 42 +++- 5 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 test/examples/docx_mimetypes.qmd create mode 100644 test/examples/typst_mimetypes.qmd diff --git a/.gitignore b/.gitignore index 72a918d..6e30287 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ Manifest.toml *.html *.ipynb *.pdf +*.typ +*.png .DS_Store \ No newline at end of file diff --git a/src/worker.jl b/src/worker.jl index b19347b..324b127 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -243,7 +243,6 @@ function worker_init(f::File) "text/plain", "text/markdown", "text/latex", - "text/html", "image/svg+xml", "image/png", ], diff --git a/test/examples/docx_mimetypes.qmd b/test/examples/docx_mimetypes.qmd new file mode 100644 index 0000000..b599c59 --- /dev/null +++ b/test/examples/docx_mimetypes.qmd @@ -0,0 +1,394 @@ +--- +title: Docx MIME types +julia: + exeflags: ["--project=integrations/CairoMakie"] +--- + +```{julia} +import CairoMakie +``` + +```{julia} +CairoMakie.scatter(1:5, 1:5) +``` + +```{julia} +struct T end + +function Base.show(io::IO, ::MIME"QuartoNotebookRunner/openxml", ::T) + print( + io, + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Overall + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Mean (SD) + + + + + + + + + + + + + + + -0.0268 + + + + ( + + + + 0.972 + + + + ) + + + + + + + + + + + + + + + + + + + + Median [Min, Max] + + + + + + + + + + + + + + + -0.117 + + + + [ + + + + -1.8 + + + + , + + + + 2.54 + + + + ] + + + + + + + + + + + + + + + + + + + + y + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Mean (SD) + + + + + + + + + + + + + + + -0.179 + + + + ( + + + + 0.911 + + + + ) + + + + + + + + + + + + + + + + + + + + Median [Min, Max] + + + + + + + + + + + + + + + -0.225 + + + + [ + + + + -2.6 + + + + , + + + + 2.35 + + + + ] + + + + + + + + + + + + + + + + + + + + + + + + """, + ) +end +``` + +```{julia} +T() +``` + +```{julia} +#| tbl-cap: Caption +T() +``` diff --git a/test/examples/typst_mimetypes.qmd b/test/examples/typst_mimetypes.qmd new file mode 100644 index 0000000..b624cf8 --- /dev/null +++ b/test/examples/typst_mimetypes.qmd @@ -0,0 +1,87 @@ +--- +title: Typst MIME types +keep-typ: true +julia: + exeflags: ["--project=integrations/CairoMakie"] +--- + +```{julia} +import CairoMakie +``` + +```{julia} +CairoMakie.scatter(1:5, 1:5) +``` + +```{julia} +struct T end + +function Base.show(io::IO, ::MIME"QuartoNotebookRunner/typst", ::T) + q = get(io, :QuartoNotebookRunner, nothing) + md = isnothing(q) ? true : isnothing(get(q.cell_options, "tbl-cap", nothing)) + print( + io, + """ + $(md ? "#" : "")[ + #import "@preview/tablex:0.0.8": tablex, cellx, hlinex + + #tablex( + columns: 4, + auto-vlines: false, + auto-hlines: false, + column-gutter: 0.25em, + hlinex(y: 0, stroke: 1pt), + cellx(colspan: 2, x: 2, y: 0, align: center + top)[#block([*Sex*])], + hlinex(y: 1, start: 2, end: 4, stroke: 0.75pt), + cellx(colspan: 1, x: 1, y: 1, align: center + top)[*Overall #linebreak() (n=10)*], + cellx(colspan: 1, x: 2, y: 1, align: center + top)[f #linebreak() (n=6)], + cellx(colspan: 1, x: 3, y: 1, align: center + top)[m #linebreak() (n=4)], + hlinex(y: 2, stroke: 0.75pt), + cellx(colspan: 1, x: 0, y: 2, align: left + top)[*Age (months)*], + cellx(colspan: 1, x: 0, y: 3, align: left + top)[#h(12.0pt)Mean (SD)], + cellx(colspan: 1, x: 1, y: 3, align: center + top)[45.6 (20.7)], + cellx(colspan: 1, x: 2, y: 3, align: center + top)[44.2 (19.1)], + cellx(colspan: 1, x: 3, y: 3, align: center + top)[47.8 (25.9)], + cellx(colspan: 1, x: 0, y: 4, align: left + top)[#h(12.0pt)Median [Min, Max]], + cellx(colspan: 1, x: 1, y: 4, align: center + top)[40.5 [24, 85]], + cellx(colspan: 1, x: 2, y: 4, align: center + top)[40.5 [24, 76]], + cellx(colspan: 1, x: 3, y: 4, align: center + top)[39.5 [27, 85]], + cellx(colspan: 1, x: 0, y: 5, align: left + top)[*Blood type*], + cellx(colspan: 1, x: 0, y: 6, align: left + top)[#h(12.0pt)0], + cellx(colspan: 1, x: 1, y: 6, align: center + top)[2 (20%)], + cellx(colspan: 1, x: 2, y: 6, align: center + top)[1 (16.7%)], + cellx(colspan: 1, x: 3, y: 6, align: center + top)[1 (25%)], + cellx(colspan: 1, x: 0, y: 7, align: left + top)[#h(12.0pt)A], + cellx(colspan: 1, x: 1, y: 7, align: center + top)[4 (40%)], + cellx(colspan: 1, x: 2, y: 7, align: center + top)[3 (50%)], + cellx(colspan: 1, x: 3, y: 7, align: center + top)[1 (25%)], + cellx(colspan: 1, x: 0, y: 8, align: left + top)[#h(12.0pt)B], + cellx(colspan: 1, x: 1, y: 8, align: center + top)[4 (40%)], + cellx(colspan: 1, x: 2, y: 8, align: center + top)[2 (33.3%)], + cellx(colspan: 1, x: 3, y: 8, align: center + top)[2 (50%)], + cellx(colspan: 1, x: 0, y: 9, align: left + top)[*Smoker*], + cellx(colspan: 1, x: 0, y: 10, align: left + top)[#h(12.0pt)false], + cellx(colspan: 1, x: 1, y: 10, align: center + top)[6 (60%)], + cellx(colspan: 1, x: 2, y: 10, align: center + top)[3 (50%)], + cellx(colspan: 1, x: 3, y: 10, align: center + top)[3 (75%)], + cellx(colspan: 1, x: 0, y: 11, align: left + top)[#h(12.0pt)true], + cellx(colspan: 1, x: 1, y: 11, align: center + top)[4 (40%)], + cellx(colspan: 1, x: 2, y: 11, align: center + top)[3 (50%)], + cellx(colspan: 1, x: 3, y: 11, align: center + top)[1 (25%)], + hlinex(y: 12, stroke: 1pt), + ) + ] + """, + ) +end +``` + +```{julia} +T() +``` + +```{julia} +# TODO: appears to not generate a caption. +#| tbl-cap: Caption +T() +``` diff --git a/test/runtests.jl b/test/runtests.jl index 1478f51..b1e07e2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -583,7 +583,9 @@ end # File-specific tests. @testset "$(relpath(each, pwd()))" begin common_tests(json) - get(() -> _ -> @test(false), tests, each)(json) + if haskey(tests, each) + tests[each](json) + end end ipynb = joinpath(examples, with_extension(each, "ipynb")) @@ -989,4 +991,42 @@ end end end end + + @testset "non-standard mime types" begin + server = QuartoNotebookRunner.Server() + expected = Dict("typst" => "```{=typst}", "docx" => "```{=openxml}") + for (format, ext) in ("typst" => "pdf", "docx" => "docx") + ipynb = joinpath(@__DIR__, "examples/$(format)_mimetypes.ipynb") + QuartoNotebookRunner.run!( + server, + joinpath(@__DIR__, "examples/$(format)_mimetypes.qmd"); + output = ipynb, + showprogress = false, + options = Dict{String,Any}( + "format" => Dict("pandoc" => Dict("to" => format)), + ), + ) + + json = JSON3.read(ipynb) + markdown = json.cells[end].outputs[1].data["text/markdown"] + @test contains(markdown, expected[format]) + + # No macOS ARM build, so just look for a local version that the dev + # should have installed. This avoids having to use rosetta2 to run + # the x86_64 version of Julia to get access to the x86_64 version of + # Quarto artifact. + quarto_bin = quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) + # Just a smoke test to make sure it runs. Use docx since it doesn't + # output a bunch of folders (html), or require a tinytex install + # (pdf). All we are doing here at the moment is ensuring quarto doesn't + # break on our notebook outputs. + if success(`$quarto_bin --version`) + @test success(`$quarto_bin render $ipynb --to $format`) + else + @error "quarto not found, skipping smoke test." + end + @test isfile(joinpath(@__DIR__, "examples/$(format)_mimetypes.$ext")) + end + end + end From a14edf2956edb1cde2f48566e5b74221087877ad Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 16:06:44 +0000 Subject: [PATCH 6/9] Adjust tests to `close!` servers more consistently Additionally, use a temp dir for mimetype test to avoid locking resources. --- test/runtests.jl | 90 +++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index b1e07e2..e144df8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -76,7 +76,6 @@ end schema = JSONSchema.Schema( open(JSON3.read, joinpath(@__DIR__, "schema/nbformat.v4.schema.json")), ) - server = QuartoNotebookRunner.Server() examples = joinpath(@__DIR__, "examples") function common_tests(json) @@ -562,6 +561,7 @@ end tests end + server = QuartoNotebookRunner.Server() for (root, dirs, files) in walkdir(examples) for each in files _, ext = splitext(each) @@ -616,6 +616,7 @@ end end end end + close!(server) # Switching exeflags within a running notebook causes it to restart so that # the new exeflags can be applied. @@ -752,6 +753,8 @@ end "data" => Dict("text/plain" => "T(\"\")"), "metadata" => Dict(), ) + + close!(server) end @testset "render" begin @@ -825,6 +828,8 @@ end output = buffer, showprogress = false, ) + + close!(server) end @testset "Invalid eval option" begin @@ -851,6 +856,8 @@ end output = buffer, showprogress = false, ) + + close!(server) end @testset "relative paths in `output`" begin @@ -913,6 +920,8 @@ end metadata = cell.outputs[1].metadata["image/png"] @test metadata.width == 625 @test metadata.height == 469 + + close!(server) end end @@ -990,43 +999,54 @@ end close!(server) end end - end - @testset "non-standard mime types" begin - server = QuartoNotebookRunner.Server() - expected = Dict("typst" => "```{=typst}", "docx" => "```{=openxml}") - for (format, ext) in ("typst" => "pdf", "docx" => "docx") - ipynb = joinpath(@__DIR__, "examples/$(format)_mimetypes.ipynb") - QuartoNotebookRunner.run!( - server, - joinpath(@__DIR__, "examples/$(format)_mimetypes.qmd"); - output = ipynb, - showprogress = false, - options = Dict{String,Any}( - "format" => Dict("pandoc" => Dict("to" => format)), - ), - ) + @testset "non-standard mime types" begin + server = QuartoNotebookRunner.Server() + expected = Dict("typst" => "```{=typst}", "docx" => "```{=openxml}") + + env = joinpath(dir, "integrations", "CairoMakie") + mkpath(env) + cp(joinpath(@__DIR__, "examples/integrations/CairoMakie"), env; force = true) + + for (format, ext) in ("typst" => "pdf", "docx" => "docx") + cd(dir) do + source = joinpath(@__DIR__, "examples/$(format)_mimetypes.qmd") + content = read(source, String) + write("$(format)_mimetypes.qmd", content) + ipynb = "$(format)_mimetypes.ipynb" + QuartoNotebookRunner.run!( + server, + "$(format)_mimetypes.qmd"; + output = ipynb, + showprogress = false, + options = Dict{String,Any}( + "format" => Dict("pandoc" => Dict("to" => format)), + ), + ) - json = JSON3.read(ipynb) - markdown = json.cells[end].outputs[1].data["text/markdown"] - @test contains(markdown, expected[format]) - - # No macOS ARM build, so just look for a local version that the dev - # should have installed. This avoids having to use rosetta2 to run - # the x86_64 version of Julia to get access to the x86_64 version of - # Quarto artifact. - quarto_bin = quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) - # Just a smoke test to make sure it runs. Use docx since it doesn't - # output a bunch of folders (html), or require a tinytex install - # (pdf). All we are doing here at the moment is ensuring quarto doesn't - # break on our notebook outputs. - if success(`$quarto_bin --version`) - @test success(`$quarto_bin render $ipynb --to $format`) - else - @error "quarto not found, skipping smoke test." + json = JSON3.read(ipynb) + markdown = json.cells[end].outputs[1].data["text/markdown"] + @test contains(markdown, expected[format]) + + # No macOS ARM build, so just look for a local version that the dev + # should have installed. This avoids having to use rosetta2 to run + # the x86_64 version of Julia to get access to the x86_64 version of + # Quarto artifact. + quarto_bin = + quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) + # Just a smoke test to make sure it runs. Use docx since it doesn't + # output a bunch of folders (html), or require a tinytex install + # (pdf). All we are doing here at the moment is ensuring quarto doesn't + # break on our notebook outputs. + if success(`$quarto_bin --version`) + @test success(`$quarto_bin render $ipynb --to $format`) + else + @error "quarto not found, skipping smoke test." + end + @test isfile("$(format)_mimetypes.$ext") + end end - @test isfile(joinpath(@__DIR__, "examples/$(format)_mimetypes.$ext")) + close!(server) end end - end From 27a755e2f6956f68d56b4947492101ce645298f5 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 16:35:31 +0000 Subject: [PATCH 7/9] debug windows failure --- test/runtests.jl | 98 ++++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index e144df8..316b7d5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -621,6 +621,55 @@ end # Switching exeflags within a running notebook causes it to restart so that # the new exeflags can be applied. mktempdir() do dir + @testset "non-standard mime types" begin + server = QuartoNotebookRunner.Server() + expected = Dict("typst" => "```{=typst}", "docx" => "```{=openxml}") + + env = joinpath(dir, "integrations", "CairoMakie") + mkpath(env) + cp(joinpath(@__DIR__, "examples/integrations/CairoMakie"), env; force = true) + + for (format, ext) in ("typst" => "pdf", "docx" => "docx") + cd(dir) do + source = joinpath(@__DIR__, "examples/$(format)_mimetypes.qmd") + content = read(source, String) + write("$(format)_mimetypes.qmd", content) + ipynb = "$(format)_mimetypes.ipynb" + QuartoNotebookRunner.run!( + server, + "$(format)_mimetypes.qmd"; + output = ipynb, + showprogress = false, + options = Dict{String,Any}( + "format" => Dict("pandoc" => Dict("to" => format)), + ), + ) + + json = JSON3.read(ipynb) + markdown = json.cells[end].outputs[1].data["text/markdown"] + @test contains(markdown, expected[format]) + + # No macOS ARM build, so just look for a local version that the dev + # should have installed. This avoids having to use rosetta2 to run + # the x86_64 version of Julia to get access to the x86_64 version of + # Quarto artifact. + quarto_bin = + quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) + # Just a smoke test to make sure it runs. Use docx since it doesn't + # output a bunch of folders (html), or require a tinytex install + # (pdf). All we are doing here at the moment is ensuring quarto doesn't + # break on our notebook outputs. + if success(`$quarto_bin --version`) + run(`$quarto_bin render $ipynb --to $format`) + @test success(`$quarto_bin render $ipynb --to $format`) + else + @error "quarto not found, skipping smoke test." + end + @test isfile("$(format)_mimetypes.$ext") + end + end + close!(server) + end @testset "exeflags notebook restart" begin content = read(joinpath(@__DIR__, "examples/stdout_exeflags.qmd"), String) cd(dir) do @@ -999,54 +1048,5 @@ end close!(server) end end - - @testset "non-standard mime types" begin - server = QuartoNotebookRunner.Server() - expected = Dict("typst" => "```{=typst}", "docx" => "```{=openxml}") - - env = joinpath(dir, "integrations", "CairoMakie") - mkpath(env) - cp(joinpath(@__DIR__, "examples/integrations/CairoMakie"), env; force = true) - - for (format, ext) in ("typst" => "pdf", "docx" => "docx") - cd(dir) do - source = joinpath(@__DIR__, "examples/$(format)_mimetypes.qmd") - content = read(source, String) - write("$(format)_mimetypes.qmd", content) - ipynb = "$(format)_mimetypes.ipynb" - QuartoNotebookRunner.run!( - server, - "$(format)_mimetypes.qmd"; - output = ipynb, - showprogress = false, - options = Dict{String,Any}( - "format" => Dict("pandoc" => Dict("to" => format)), - ), - ) - - json = JSON3.read(ipynb) - markdown = json.cells[end].outputs[1].data["text/markdown"] - @test contains(markdown, expected[format]) - - # No macOS ARM build, so just look for a local version that the dev - # should have installed. This avoids having to use rosetta2 to run - # the x86_64 version of Julia to get access to the x86_64 version of - # Quarto artifact. - quarto_bin = - quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) - # Just a smoke test to make sure it runs. Use docx since it doesn't - # output a bunch of folders (html), or require a tinytex install - # (pdf). All we are doing here at the moment is ensuring quarto doesn't - # break on our notebook outputs. - if success(`$quarto_bin --version`) - @test success(`$quarto_bin render $ipynb --to $format`) - else - @error "quarto not found, skipping smoke test." - end - @test isfile("$(format)_mimetypes.$ext") - end - end - close!(server) - end end end From 53dc1e6c931d913a2939f2fb1603a3480d892cb7 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 17:10:22 +0000 Subject: [PATCH 8/9] Conditionally call `quarto`, skip Windows for now --- test/runtests.jl | 64 +++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 316b7d5..8fcac82 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -596,20 +596,22 @@ end showprogress = false, ) - # No macOS ARM build, so just look for a local version that the dev - # should have installed. This avoids having to use rosetta2 to run - # the x86_64 version of Julia to get access to the x86_64 version of - # Quarto artifact. - quarto_bin = - quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) - # Just a smoke test to make sure it runs. Use docx since it doesn't - # output a bunch of folders (html), or require a tinytex install - # (pdf). All we are doing here at the moment is ensuring quarto doesn't - # break on our notebook outputs. - if success(`$quarto_bin --version`) - @test success(`$quarto_bin render $ipynb --to docx`) - else - @error "quarto not found, skipping smoke test." + if !Sys.iswindows() + # No macOS ARM build, so just look for a local version that the dev + # should have installed. This avoids having to use rosetta2 to run + # the x86_64 version of Julia to get access to the x86_64 version of + # Quarto artifact. + quarto_bin = + quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) + # Just a smoke test to make sure it runs. Use docx since it doesn't + # output a bunch of folders (html), or require a tinytex install + # (pdf). All we are doing here at the moment is ensuring quarto doesn't + # break on our notebook outputs. + if success(`$quarto_bin --version`) + @test success(`$quarto_bin render $ipynb --to docx`) + else + @error "quarto not found, skipping smoke test." + end end QuartoNotebookRunner.close!(server, each) @@ -649,23 +651,25 @@ end markdown = json.cells[end].outputs[1].data["text/markdown"] @test contains(markdown, expected[format]) - # No macOS ARM build, so just look for a local version that the dev - # should have installed. This avoids having to use rosetta2 to run - # the x86_64 version of Julia to get access to the x86_64 version of - # Quarto artifact. - quarto_bin = - quarto_jll.is_available() ? quarto_jll.quarto() : setenv(`quarto`) - # Just a smoke test to make sure it runs. Use docx since it doesn't - # output a bunch of folders (html), or require a tinytex install - # (pdf). All we are doing here at the moment is ensuring quarto doesn't - # break on our notebook outputs. - if success(`$quarto_bin --version`) - run(`$quarto_bin render $ipynb --to $format`) - @test success(`$quarto_bin render $ipynb --to $format`) - else - @error "quarto not found, skipping smoke test." + if !Sys.iswindows() + # No macOS ARM build, so just look for a local version that the dev + # should have installed. This avoids having to use rosetta2 to run + # the x86_64 version of Julia to get access to the x86_64 version of + # Quarto artifact. + quarto_bin = + quarto_jll.is_available() ? quarto_jll.quarto() : + setenv(`quarto`) + # Just a smoke test to make sure it runs. Use docx since it doesn't + # output a bunch of folders (html), or require a tinytex install + # (pdf). All we are doing here at the moment is ensuring quarto doesn't + # break on our notebook outputs. + if success(`$quarto_bin --version`) + @test success(`$quarto_bin render $ipynb --to $format`) + else + @error "quarto not found, skipping smoke test." + end + @test isfile("$(format)_mimetypes.$ext") end - @test isfile("$(format)_mimetypes.$ext") end end close!(server) From 395c037e78ab51c8614c3b06b038b0d7ae3f048d Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 1 Feb 2024 17:43:20 +0000 Subject: [PATCH 9/9] Add `label` to code blocks --- test/examples/docx_mimetypes.qmd | 1 + test/examples/typst_mimetypes.qmd | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/examples/docx_mimetypes.qmd b/test/examples/docx_mimetypes.qmd index b599c59..969a5f5 100644 --- a/test/examples/docx_mimetypes.qmd +++ b/test/examples/docx_mimetypes.qmd @@ -389,6 +389,7 @@ T() ``` ```{julia} +#| label: tbl-table-name #| tbl-cap: Caption T() ``` diff --git a/test/examples/typst_mimetypes.qmd b/test/examples/typst_mimetypes.qmd index b624cf8..9769c7c 100644 --- a/test/examples/typst_mimetypes.qmd +++ b/test/examples/typst_mimetypes.qmd @@ -22,7 +22,7 @@ function Base.show(io::IO, ::MIME"QuartoNotebookRunner/typst", ::T) print( io, """ - $(md ? "#" : "")[ + $(md ? "#[" : "") #import "@preview/tablex:0.0.8": tablex, cellx, hlinex #tablex( @@ -70,7 +70,7 @@ function Base.show(io::IO, ::MIME"QuartoNotebookRunner/typst", ::T) cellx(colspan: 1, x: 3, y: 11, align: center + top)[1 (25%)], hlinex(y: 12, stroke: 1pt), ) - ] + $(md ? "]" : "") """, ) end @@ -81,7 +81,7 @@ T() ``` ```{julia} -# TODO: appears to not generate a caption. +#| label: tbl-table-name #| tbl-cap: Caption T() ```