Update pulldown_cmark dep to v0.10, and add pulldown_cmark_escape dep.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## 0.19.0 (unreleased)
 
+- Updates the pulldown-cmark dependency to v0.10.0. This improves footnote handling, and may also introduce some minor behavior changes such as reducing the amount of unnecessary HTML-escaping of text content.
 
 ## 0.18.0 (2023-12-18)
 

components/libs/Cargo.toml

@@ -21,7 +21,8 @@ nom-bibtex = "0.5"
 num-format = "0.4"
 once_cell = "1"
 percent-encoding = "2"
-pulldown-cmark = { version = "0.9", default-features = false, features = ["simd"]  }
+pulldown-cmark = { version = "0.10", default-features = false, features = ["html", "simd"] }
+pulldown-cmark-escape = { version = "0.10", default-features = false }
 quickxml_to_serde = "0.5"
 rayon = "1"
 regex = "1"

components/libs/src/lib.rs

@@ -23,6 +23,7 @@ pub use num_format;
 pub use once_cell;
 pub use percent_encoding;
 pub use pulldown_cmark;
+pub use pulldown_cmark_escape;
 pub use quickxml_to_serde;
 pub use rayon;
 pub use regex;

components/markdown/src/markdown.rs

@@ -4,19 +4,20 @@ use errors::bail;
 use libs::gh_emoji::Replacer as EmojiReplacer;
 use libs::once_cell::sync::Lazy;
 use libs::pulldown_cmark as cmark;
+use libs::pulldown_cmark_escape as cmark_escape;
 use libs::tera;
 use utils::net::is_external_link;
 
 use crate::context::RenderContext;
 use errors::{Context, Error, Result};
-use libs::pulldown_cmark::escape::escape_html;
+use libs::pulldown_cmark_escape::escape_html;
 use libs::regex::{Regex, RegexBuilder};
 use utils::site::resolve_internal_link;
 use utils::slugs::slugify_anchors;
 use utils::table_of_contents::{make_table_of_contents, Heading};
 use utils::types::InsertAnchor;
 
-use self::cmark::{Event, LinkType, Options, Parser, Tag};
+use self::cmark::{Event, LinkType, Options, Parser, Tag, TagEnd};
 use crate::codeblock::{CodeBlock, FenceSettings};
 use crate::shortcode::{Shortcode, SHORTCODE_PLACEHOLDER};
 
@@ -220,15 +221,15 @@ fn get_heading_refs(events: &[Event]) -> Vec<HeadingRef> {
 
     for (i, event) in events.iter().enumerate() {
         match event {
-            Event::Start(Tag::Heading(level, anchor, classes)) => {
+            Event::Start(Tag::Heading { level, id, classes, .. }) => {
                 heading_refs.push(HeadingRef::new(
                     i,
                     *level as u32,
-                    anchor.map(|a| a.to_owned()),
+                    id.clone().map(|a| a.to_string()),
                     &classes.iter().map(|x| x.to_string()).collect::<Vec<_>>(),
                 ));
             }
-            Event::End(Tag::Heading(_, _, _)) => {
+            Event::End(TagEnd::Heading { .. }) => {
                 heading_refs.last_mut().expect("Heading end before start?").end_idx = i;
             }
             _ => (),
@@ -254,6 +255,10 @@ pub fn markdown_to_html(
     let mut error = None;
 
     let mut code_block: Option<CodeBlock> = None;
+    // Indicates whether we're in the middle of parsing a text node which will be placed in an HTML
+    // attribute, and which hence has to be escaped using escape_html rather than push_html's
+    // default HTML body escaping for text nodes.
+    let mut inside_attribute = false;
 
     let mut headings: Vec<Heading> = vec![];
     let mut internal_links = Vec::new();
@@ -294,12 +299,19 @@ pub fn markdown_to_html(
 
                         // we have some text before the shortcode, push that first
                         if $range.start != sc_span.start {
-                            let content = $text[($range.start - orig_range_start)
-                                ..(sc_span.start - orig_range_start)]
-                                .to_string()
-                                .into();
+                            let content: cmark::CowStr<'_> =
+                                $text[($range.start - orig_range_start)
+                                    ..(sc_span.start - orig_range_start)]
+                                    .to_string()
+                                    .into();
                             events.push(if $is_text {
-                                Event::Text(content)
+                                if inside_attribute {
+                                    let mut buffer = "".to_string();
+                                    escape_html(&mut buffer, content.as_ref()).unwrap();
+                                    Event::Html(buffer.into())
+                                } else {
+                                    Event::Text(content)
+                                }
                             } else {
                                 Event::Html(content)
                             });
@@ -370,7 +382,13 @@ pub fn markdown_to_html(
                         };
 
                         if !contains_shortcode(text.as_ref()) {
-                            events.push(Event::Text(text));
+                            if inside_attribute {
+                                let mut buffer = "".to_string();
+                                escape_html(&mut buffer, text.as_ref()).unwrap();
+                                events.push(Event::Html(buffer.into()));
+                            } else {
+                                events.push(Event::Text(text));
+                            }
                             continue;
                         }
 
@@ -386,7 +404,7 @@ pub fn markdown_to_html(
                     code_block = Some(block);
                     events.push(Event::Html(begin.into()));
                 }
-                Event::End(Tag::CodeBlock(_)) => {
+                Event::End(TagEnd::CodeBlock { .. }) => {
                     if let Some(ref mut code_block) = code_block {
                         let html = code_block.highlight(&accumulated_block);
                         events.push(Event::Html(html.into()));
@@ -397,44 +415,53 @@ pub fn markdown_to_html(
                     code_block = None;
                     events.push(Event::Html("</code></pre>\n".into()));
                 }
-                Event::Start(Tag::Image(link_type, src, title)) => {
-                    let link = if is_colocated_asset_link(&src) {
-                        let link = format!("{}{}", context.current_page_permalink, &*src);
+                Event::Start(Tag::Image { link_type, dest_url, title, id }) => {
+                    let link = if is_colocated_asset_link(&dest_url) {
+                        let link = format!("{}{}", context.current_page_permalink, &*dest_url);
                         link.into()
                     } else {
-                        src
+                        dest_url
                     };
 
                     events.push(if lazy_async_image {
                         let mut img_before_alt: String = "<img src=\"".to_string();
-                        cmark::escape::escape_href(&mut img_before_alt, &link)
+                        cmark_escape::escape_href(&mut img_before_alt, &link)
                             .expect("Could not write to buffer");
                         if !title.is_empty() {
                             img_before_alt
                                 .write_str("\" title=\"")
                                 .expect("Could not write to buffer");
-                            cmark::escape::escape_href(&mut img_before_alt, &title)
+                            cmark_escape::escape_href(&mut img_before_alt, &title)
                                 .expect("Could not write to buffer");
                         }
                         img_before_alt.write_str("\" alt=\"").expect("Could not write to buffer");
+                        inside_attribute = true;
                         Event::Html(img_before_alt.into())
                     } else {
-                        Event::Start(Tag::Image(link_type, link, title))
+                        inside_attribute = false;
+                        Event::Start(Tag::Image { link_type, dest_url: link, title, id })
                     });
                 }
-                Event::End(Tag::Image(..)) => events.push(if lazy_async_image {
+                Event::End(TagEnd::Image) => events.push(if lazy_async_image {
                     Event::Html("\" loading=\"lazy\" decoding=\"async\" />".into())
                 } else {
                     event
                 }),
-                Event::Start(Tag::Link(link_type, link, title)) if link.is_empty() => {
+                Event::Start(Tag::Link { link_type, dest_url, title, id })
+                    if dest_url.is_empty() =>
+                {
                     error = Some(Error::msg("There is a link that is missing a URL"));
-                    events.push(Event::Start(Tag::Link(link_type, "#".into(), title)));
+                    events.push(Event::Start(Tag::Link {
+                        link_type,
+                        dest_url: "#".into(),
+                        title,
+                        id,
+                    }));
                 }
-                Event::Start(Tag::Link(link_type, link, title)) => {
+                Event::Start(Tag::Link { link_type, dest_url, title, id }) => {
                     let fixed_link = match fix_link(
                         link_type,
-                        &link,
+                        &dest_url,
                         context,
                         &mut internal_links,
                         &mut external_links,
@@ -448,12 +475,12 @@ pub fn markdown_to_html(
                     };
 
                     events.push(
-                        if is_external_link(&link)
+                        if is_external_link(&dest_url)
                             && context.config.markdown.has_external_link_tweaks()
                         {
                             let mut escaped = String::new();
                             // write_str can fail but here there are no reasons it should (afaik?)
-                            cmark::escape::escape_href(&mut escaped, &link)
+                            cmark_escape::escape_href(&mut escaped, &dest_url)
                                 .expect("Could not write to buffer");
                             Event::Html(
                                 context
@@ -463,7 +490,12 @@ pub fn markdown_to_html(
                                     .into(),
                             )
                         } else {
-                            Event::Start(Tag::Link(link_type, fixed_link.into(), title))
+                            Event::Start(Tag::Link {
+                                link_type,
+                                dest_url: fixed_link.into(),
+                                title,
+                                id,
+                            })
                         },
                     )
                 }
@@ -485,7 +517,7 @@ pub fn markdown_to_html(
 
                     events.push(event);
                 }
-                Event::End(Tag::Paragraph) => {
+                Event::End(TagEnd::Paragraph) => {
                     events.push(if stop_next_end_p {
                         stop_next_end_p = false;
                         Event::Html("".into())

components/markdown/tests/snapshots/markdown__all_markdown_features_integration.snap

@@ -1,8 +1,6 @@
 ---
-source: components/rendering/tests/markdown.rs
-assertion_line: 358
+source: components/markdown/tests/markdown.rs
 expression: body
-
 ---
 <!-- Adapted from https://markdown-it.github.io/ -->
 <h1 id="h1-heading">h1 Heading</h1>
@@ -83,7 +81,7 @@ line 1 of code
 line 2 of code
 line 3 of code
 </code></pre>
-<p>Block code &quot;fences&quot;</p>
+<p>Block code "fences"</p>
 <pre><code>Sample text here...
 </code></pre>
 <p>Syntax highlighting</p>

components/markdown/tests/snapshots/markdown__can_handle_heading_ids-2.snap

@@ -1,12 +1,10 @@
 ---
-source: components/rendering/tests/markdown.rs
-assertion_line: 84
+source: components/markdown/tests/markdown.rs
 expression: body
-
 ---
 <h1 id="Hello">Hello</h1>
 <h1 id="Hello-1">Hello</h1>
-<h1 id="L'écologie_et_vous">L'écologie et vous</h1>
+<h1 id="L&#39;écologie_et_vous">L'écologie et vous</h1>
 <h1 id="hello">Hello</h1>
 <h1 id="hello">Hello</h1>
 <h1 id="Something_else">Hello</h1>
@@ -22,6 +20,6 @@ expression: body
 <h1 id="text__there">text <sup class="footnote-reference"><a href="#1">1</a></sup> there</h1>
 <div class="footnote-definition" id="1"><sup class="footnote-definition-label">1</sup>
 <p>footnote</p>
-<h1 id="classes" class="bold another">Classes</h1>
 </div>
+<h1 id="classes" class="bold another">Classes</h1>
 

components/markdown/tests/snapshots/markdown__can_handle_heading_ids.snap

@@ -1,8 +1,6 @@
 ---
-source: components/rendering/tests/markdown.rs
-assertion_line: 79
+source: components/markdown/tests/markdown.rs
 expression: body
-
 ---
 <h1 id="hello-1">Hello</h1>
 <h1 id="hello-2">Hello</h1>
@@ -22,6 +20,6 @@ expression: body
 <h1 id="text-there">text <sup class="footnote-reference"><a href="#1">1</a></sup> there</h1>
 <div class="footnote-definition" id="1"><sup class="footnote-definition-label">1</sup>
 <p>footnote</p>
-<h1 id="classes" class="bold another">Classes</h1>
 </div>
+<h1 id="classes" class="bold another">Classes</h1>
 

components/markdown/tests/snapshots/shortcodes__doesnt_render_ignored_shortcodes.snap

@@ -1,8 +1,6 @@
 ---
-source: components/rendering/tests/shortcodes.rs
-assertion_line: 104
+source: components/markdown/tests/shortcodes.rs
 expression: body
-
 ---
-<p>{{ youtube(id=&quot;w7Ft2ymGmfc&quot;) }}</p>
+<p>{{ youtube(id="w7Ft2ymGmfc") }}</p>