Use quarto-provided markdown if present (#154)

* use quarto-provided markdown if present

* format

* remove stray `display` call

* add quarto integration test

* refactor to not need temp file

.github/workflows/CI.yml

@@ -80,6 +80,11 @@ jobs:
       - run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
         if: matrix.os == 'ubuntu-latest'
 
+      # TODO: use quarto_jll for integration tests once modern enough versions are available
+      - uses: quarto-dev/quarto-actions/setup@v2
+        with:
+          version: pre-release
+
       - uses: julia-actions/julia-buildpkg@90dd6f23eb49626e4e6612cb9d64d456f86e6a1c
       - uses: julia-actions/julia-runtest@79a7e100883947123f8263c5f06e6c0ea3eb972f
         with:

src/server.jl

@@ -116,20 +116,23 @@ Evaluate the code and markdown in `f` and return a vector of cells with the
 results in all available mimetypes.
 
 `output` can be a file path, or an IO stream.
+`markdown` can be used to pass an override for the file content, which is used
+    to pass the modified markdown that quarto creates after resolving shortcodes
 """
 function evaluate!(
     f::File,
     output::Union{AbstractString,IO,Nothing} = nothing;
     showprogress = true,
     options::Union{String,Dict{String,Any}} = Dict{String,Any}(),
     chunk_callback = (i, n, c) -> nothing,
+    markdown::Union{String,Nothing} = nothing,
 )
     _check_output_dst(output)
 
     options = _parsed_options(options)
     path = abspath(f.path)
     if isfile(path)
-        raw_chunks, file_frontmatter = raw_text_chunks(f)
+        raw_chunks, file_frontmatter = raw_text_chunks(f, markdown)
         merged_options = _extract_relevant_options(file_frontmatter, options)
         cells =
             evaluate_raw_cells!(f, raw_chunks, merged_options; showprogress, chunk_callback)
@@ -285,7 +288,9 @@ write_json(::Nothing, data) = data
 Return a vector of raw markdown and code chunks from `file` ready for evaluation
 by `evaluate_raw_cells!`.
 """
-raw_text_chunks(file::File) = raw_text_chunks(file.path)
+raw_text_chunks(file::File, ::Nothing) = raw_text_chunks(file.path)
+raw_text_chunks(file::File, markdown::String) =
+    raw_markdown_chunks_from_string(file.path, markdown)
 
 function raw_text_chunks(path::String)
     endswith(path, ".qmd") && return raw_markdown_chunks(path)
@@ -299,85 +304,74 @@ end
 Return a vector of raw markdown and code chunks from `file` ready
 for evaluation by `evaluate_raw_cells!`.
 """
-raw_markdown_chunks(file::File) = raw_markdown_chunks(file.path)
-
-function raw_markdown_chunks(path::String)
-    if !endswith(path, ".qmd")
-        throw(ArgumentError("file is not a quarto markdown file: $(path)"))
-    end
-
-    if isfile(path)
-        raw_chunks = []
-        ast = open(Parser(), path)
-        source_lines = readlines(path)
-        terminal_line = 1
-        code_cells = false
-        for (node, enter) in ast
-            if enter && (is_julia_toplevel(node) || is_r_toplevel(node))
-                code_cells = true
-                line = node.sourcepos[1][1]
-                markdown = join(source_lines[terminal_line:(line-1)], "\n")
-                push!(
-                    raw_chunks,
-                    (
-                        type = :markdown,
-                        source = markdown,
-                        file = path,
-                        line = terminal_line,
-                    ),
-                )
-                terminal_line = node.sourcepos[2][1] + 1
-
-                # currently, the only execution-relevant cell option is `eval` which controls if a code block is executed or not.
-                # this option could in the future also include a vector of line numbers, which knitr supports.
-                # all other options seem to be quarto-rendering related, like where to put figure captions etc.
-                source = node.literal
-                cell_options = extract_cell_options(source; file = path, line = line)
-                evaluate = get(cell_options, "eval", true)
-                if !(evaluate isa Bool)
-                    error(
-                        "Cannot handle an `eval` code cell option with value $(repr(evaluate)), only true or false.",
-                    )
-                end
-                language =
-                    is_julia_toplevel(node) ? :julia :
-                    is_r_toplevel(node) ? :r : error("Unhandled code block language")
-                push!(
-                    raw_chunks,
-                    (
-                        type = :code,
-                        language = language,
-                        source,
-                        file = path,
-                        line,
-                        evaluate,
-                        cell_options,
-                    ),
-                )
-            end
-        end
-        if terminal_line <= length(source_lines)
-            markdown = join(source_lines[terminal_line:end], "\n")
+raw_markdown_chunks(file::File) =
+    endswith(path, ".qmd") ? raw_markdown_chunks(file.path) :
+    throw(ArgumentError("file is not a quarto markdown file: $(path)"))
+raw_markdown_chunks(path::String) =
+    raw_markdown_chunks_from_string(path, read(path, String))
+
+function raw_markdown_chunks_from_string(path::String, markdown::String)
+    raw_chunks = []
+    pars = Parser()
+    ast = pars(markdown; source = path)
+    source_lines = collect(eachline(IOBuffer(markdown)))
+    terminal_line = 1
+    code_cells = false
+    for (node, enter) in ast
+        if enter && (is_julia_toplevel(node) || is_r_toplevel(node))
+            code_cells = true
+            line = node.sourcepos[1][1]
+            md = join(source_lines[terminal_line:(line-1)], "\n")
             push!(
                 raw_chunks,
-                (type = :markdown, source = markdown, file = path, line = terminal_line),
+                (type = :markdown, source = md, file = path, line = terminal_line),
             )
-        end
-
-        # The case where the notebook has no code cells.
-        if isempty(raw_chunks) && !code_cells
+            terminal_line = node.sourcepos[2][1] + 1
+
+            # currently, the only execution-relevant cell option is `eval` which controls if a code block is executed or not.
+            # this option could in the future also include a vector of line numbers, which knitr supports.
+            # all other options seem to be quarto-rendering related, like where to put figure captions etc.
+            source = node.literal
+            cell_options = extract_cell_options(source; file = path, line = line)
+            evaluate = get(cell_options, "eval", true)
+            if !(evaluate isa Bool)
+                error(
+                    "Cannot handle an `eval` code cell option with value $(repr(evaluate)), only true or false.",
+                )
+            end
+            language =
+                is_julia_toplevel(node) ? :julia :
+                is_r_toplevel(node) ? :r : error("Unhandled code block language")
             push!(
                 raw_chunks,
-                (type = :markdown, source = read(path, String), file = path, line = 1),
+                (
+                    type = :code,
+                    language = language,
+                    source,
+                    file = path,
+                    line,
+                    evaluate,
+                    cell_options,
+                ),
             )
         end
+    end
+    if terminal_line <= length(source_lines)
+        md = join(source_lines[terminal_line:end], "\n")
+        push!(
+            raw_chunks,
+            (type = :markdown, source = md, file = path, line = terminal_line),
+        )
+    end
 
-        frontmatter = _recursive_merge(default_frontmatter(), CommonMark.frontmatter(ast))
-
-        return raw_chunks, frontmatter
-    else
-        throw(ArgumentError("file does not exist: $(path)"))
+    # The case where the notebook has no code cells.
+    if isempty(raw_chunks) && !code_cells
+        push!(raw_chunks, (type = :markdown, source = markdown, file = path, line = 1))
     end
+
+    frontmatter = _recursive_merge(default_frontmatter(), CommonMark.frontmatter(ast))
+
+    return raw_chunks, frontmatter
 end
 
 _recursive_merge(x::AbstractDict...) = merge(_recursive_merge, x...)
@@ -1067,6 +1061,7 @@ function run!(
     server::Server,
     path::AbstractString;
     output::Union{AbstractString,IO,Nothing} = nothing,
+    markdown::Union{Nothing,String} = nothing,
     showprogress::Bool = true,
     options::Union{String,Dict{String,Any}} = Dict{String,Any}(),
     chunk_callback = (i, n, c) -> nothing,
@@ -1076,7 +1071,7 @@ function run!(
             close(file.timeout_timer)
             file.timeout_timer = nothing
         end
-        result = evaluate!(file, output; showprogress, options, chunk_callback)
+        result = evaluate!(file, output; showprogress, options, markdown, chunk_callback)
         if file.timeout > 0
             file.timeout_timer = Timer(file.timeout) do _
                 close!(server, file.path)

src/socket.jl

@@ -254,6 +254,7 @@ function _handle_response(
 
     if type == "run"
         options = _get_options(request.content)
+        markdown = _get_markdown(options)
 
         function chunk_callback(i, n, chunk)
             _write_json(
@@ -269,7 +270,16 @@ function _handle_response(
         end
 
         result = try
-            (; notebook = run!(notebooks, file; options, showprogress, chunk_callback))
+            (;
+                notebook = run!(
+                    notebooks,
+                    file;
+                    options,
+                    markdown,
+                    showprogress,
+                    chunk_callback,
+                )
+            )
         catch error
             _log_error("Failed to run notebook: $file", error, catch_backtrace())
         end
@@ -356,6 +366,17 @@ _get_file(content::String) = content
 _get_options(content::Dict) = get(Dict{String,Any}, content, "options")
 _get_options(::String) = Dict{String,Any}()
 
+function _get_nested(d::Dict, keys...)
+    _d = d
+    for key in keys
+        _d = get(_d, key, nothing)
+        _d === nothing && return
+    end
+    return _d
+end
+_get_markdown(options::Dict)::Union{Nothing,String} =
+    _get_nested(options, "target", "markdown", "value")
+
 # Compat:
 
 if !isdefined(Base, :errormonitor)

test/examples/quarto_integration/to_include.qmd

@@ -0,0 +1,3 @@
+```{julia}
+x = 1
+```

test/examples/quarto_integration/with_include.qmd

@@ -0,0 +1,10 @@
+---
+engine: julia
+---
+
+{{< include to_include.qmd >}}
+
+```{julia}
+y = x + 1
+println("y = $y")
+```

test/testsets/quarto_integration/includes.jl

@@ -0,0 +1,16 @@
+include("../../utilities/prelude.jl")
+
+@testset "quarto includes" begin
+    file =
+        joinpath(@__DIR__, "..", "..", "examples", "quarto_integration", "with_include.qmd")
+    # TODO: use quarto_jll for integration tests once modern enough versions are available
+    cmd = addenv(
+        `quarto render $file --to md`,
+        "QUARTO_JULIA_PROJECT" => normpath(joinpath(@__DIR__, "..", "..", "..")),
+    )
+    run(cmd)
+    outputfile =
+        joinpath(@__DIR__, "..", "..", "examples", "quarto_integration", "with_include.md")
+    @test occursin("y = 2", read(outputfile, String))
+    rm(outputfile)
+end