bug(renderer): source highlighter should get unadulterated string

Ths fix for this involved some refactoring of the highlighting work
to separate out the calloouts (we embed them using reserved Unicode so
that we can find them again and replace them inline).

Chroma automatically escapes the code for HTML safetly already, so
we defer to it's rules, when we use it.  We still have to do this
ourselves when not using Chroma.

Fixes bytesparadise#721

LIMITATIONS.adoc

@@ -136,6 +136,10 @@ is treated as an alias.  Chroma supports all the standard pygments styles, as we
 majority of the same source code languages. However some more esoteric languages might not be supported.
 See https://github.com/alecthomas/chroma#supported-languages[Chroma's documentation] for details.
 
+== Callouts
+
+Libasciidoc does not support font or image based callouts yet.
+
 == Math
 
 MathML and equations (`[stem]` blocks) are not supported yet.

pkg/renderer/sgml/delimited_block.go

@@ -2,6 +2,7 @@ package sgml
 
 import (
 	"bytes"
+	"strconv"
 	"strings"
 
 	"github.com/alecthomas/chroma"
@@ -107,6 +108,71 @@ func (r *sgmlRenderer) renderListingBlock(ctx *renderer.Context, b types.Delimit
 	return result.String(), err
 }
 
+func (r *sgmlRenderer) renderSourceLine(w *strings.Builder, item interface{}) {
+	switch item := item.(type) {
+	case types.VerbatimLine:
+		w.WriteString(item.Content)
+		for _, co := range item.Callouts {
+			// We inject an escaped sequence for now, which we can replace later with
+			// a fully rendered version of the callout. We use two non-characters
+			// (reserved for this kind of use by Unicode) to bracket callouts, allowing
+			// us to find them again in post-processing..
+			w.WriteString("\ufdd0")
+			w.WriteString(strconv.Itoa(co.Ref))
+			w.WriteString("\ufdd1")
+		}
+	case types.StringElement:
+		w.WriteString(item.Content)
+	case []interface{}:
+		for _, sub := range item {
+			r.renderSourceLine(w, sub)
+		}
+	}
+}
+
+func (r *sgmlRenderer) renderCalloutRef(co types.Callout) (string, error) {
+	result := &strings.Builder{}
+	err := r.calloutRef.Execute(result, co)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to render callout number")
+	}
+	return result.String(), nil
+}
+
+func (r *sgmlRenderer) renderSourceCallouts(source string) (string, error) {
+	result := &strings.Builder{}
+	num := 0
+	co := false
+	for _, ch := range source {
+		if co {
+			if ch >= '0' && ch <= '9' {
+				num *= 10
+				num += int(ch - '0')
+				continue
+			}
+			if ch == '\ufdd1' {
+				s, err := r.renderCalloutRef(types.Callout{Ref: num})
+				if err != nil {
+					return "", errors.Wrap(err, "unable to render source block")
+				}
+				result.WriteString(s)
+				co = false
+				continue
+			}
+			// unexpected character - just copy it to output.
+			result.WriteRune(ch)
+		}
+		if ch == '\ufdd0' {
+			// start of integer
+			num = 0
+			co = true
+			continue
+		}
+		result.WriteRune(ch)
+	}
+	return result.String(), nil
+}
+
 func (r *sgmlRenderer) renderSourceBlock(ctx *renderer.Context, b types.DelimitedBlock) (string, error) {
 	previousWithinDelimitedBlock := ctx.WithinDelimitedBlock
 	previousIncludeBlankLine := ctx.IncludeBlankLine
@@ -119,15 +185,20 @@ func (r *sgmlRenderer) renderSourceBlock(ctx *renderer.Context, b types.Delimite
 	// first, render the content
 
 	elements := discardTrailingBlankLines(b.Elements)
-	content, err := r.renderElements(ctx, elements)
-
-	if err != nil {
-		return "", err
-	}
 
 	highlighter, _ := ctx.Attributes.GetAsString(types.AttrSyntaxHighlighter)
 	language, found := b.Attributes.GetAsString(types.AttrLanguage)
+	content := ""
 	if found && (highlighter == "chroma" || highlighter == "pygments") {
+
+		source := &strings.Builder{}
+		for i, line := range elements {
+			if i > 0 {
+				source.WriteRune('\n')
+			}
+			r.renderSourceLine(source, line)
+		}
+
 		// using github.com/alecthomas/chroma to highlight the content
 		contentBuf := &strings.Builder{}
 		lexer := lexers.Get(language)
@@ -140,7 +211,8 @@ func (r *sgmlRenderer) renderSourceBlock(ctx *renderer.Context, b types.Delimite
 		if s, found := ctx.Attributes.GetAsString(highlighter + "-style"); found {
 			style = styles.Get(s)
 		}
-		iterator, err := lexer.Tokenise(nil, content)
+		// iterator, err := lexer.Tokenise(nil, content)
+		iterator, err := lexer.Tokenise(nil, source.String())
 		if err != nil {
 			return "", err
 		}
@@ -163,10 +235,21 @@ func (r *sgmlRenderer) renderSourceBlock(ctx *renderer.Context, b types.Delimite
 			return "", err
 		}
 		content = contentBuf.String()
+		content, err = r.renderSourceCallouts(content)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		elements := discardTrailingBlankLines(b.Elements)
+		if s, err := r.renderElements(ctx, elements); err != nil {
+			return "", err
+		} else {
+			content = s
+		}
 	}
 
 	result := &bytes.Buffer{}
-	err = r.sourceBlock.Execute(result, struct {
+	err := r.sourceBlock.Execute(result, struct {
 		ID                sanitized
 		Title             sanitized
 		Roles             sanitized
@@ -181,6 +264,7 @@ func (r *sgmlRenderer) renderSourceBlock(ctx *renderer.Context, b types.Delimite
 		Language:          language,
 		Content:           content,
 	})
+
 	return result.String(), err
 }
 

pkg/renderer/sgml/html5/callout_list.go

@@ -12,4 +12,7 @@ const (
 
 	// NB: The items are numbered sequentially.
 	calloutListItemTmpl = "<li>\n{{ .Content }}</li>\n"
+
+	// This should probably have been a <span>, but for compatibility we use <b>
+	calloutRefTmpl = "<b class=\"conum\">({{ .Ref }})</b>"
 )

pkg/renderer/sgml/html5/delimited_block_test.go

@@ -357,6 +357,36 @@ end</code></pre>
 			Expect(RenderHTML(source)).To(MatchHTML(expected))
 		})
 
+		It("with C content, and callout", func() {
+			source := `:source-highlighter: chroma
+[source, c]
+----
+#include <stdio.h>
+
+printf("Hello world!\n"); // <1>
+<a>link</a>
+----
+<1> A greeting
+`
+			expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-cp">#include</span> <span class="tok-cpf">&lt;stdio.h&gt;</span><span class="tok-cp">
+</span><span class="tok-cp"></span>
+<span class="tok-n">printf</span><span class="tok-p">(</span><span class="tok-s">&#34;Hello world!</span><span class="tok-se">\n</span><span class="tok-s">&#34;</span><span class="tok-p">);</span> <span class="tok-c1">// <b class="conum">(1)</b>
+</span><span class="tok-c1"></span><span class="tok-o">&lt;</span><span class="tok-n">a</span><span class="tok-o">&gt;</span><span class="tok-n">link</span><span class="tok-o">&lt;/</span><span class="tok-n">a</span><span class="tok-o">&gt;</span></code></pre>
+</div>
+</div>
+<div class="colist arabic">
+<ol>
+<li>
+<p>A greeting</p>
+</li>
+</ol>
+</div>
+`
+			Expect(RenderHTML(source)).To(MatchHTML(expected))
+		})
+
 		It("with other content", func() {
 			source := `----
   a<<b

pkg/renderer/sgml/html5/templates.go

@@ -18,6 +18,7 @@ var templates = sgml.Templates{
 	BoldText:                  boldTextTmpl,
 	CalloutList:               calloutListTmpl,
 	CalloutListItem:           calloutListItemTmpl,
+	CalloutRef:                calloutRefTmpl,
 	DelimitedBlockParagraph:   delimitedBlockParagraphTmpl,
 	DocumentDetails:           documentDetailsTmpl,
 	DocumentAuthorDetails:     documentAuthorDetailsTmpl,

pkg/renderer/sgml/sgml_renderer.go

@@ -18,6 +18,7 @@ type sgmlRenderer struct {
 	boldText                  *textTemplate
 	calloutList               *textTemplate
 	calloutListItem           *textTemplate
+	calloutRef                *textTemplate
 	delimitedBlockParagraph   *textTemplate
 	documentDetails           *textTemplate
 	documentAuthorDetails     *textTemplate
@@ -96,6 +97,7 @@ func (r *sgmlRenderer) prepareTemplates() error {
 		r.boldText, err = r.newTemplate("bold-text", tmpls.BoldText, err)
 		r.calloutList, err = r.newTemplate("callout-list", tmpls.CalloutList, err)
 		r.calloutListItem, err = r.newTemplate("callout-list-item", tmpls.CalloutListItem, err)
+		r.calloutRef, err = r.newTemplate("callout-ref", tmpls.CalloutRef, err)
 		r.delimitedBlockParagraph, err = r.newTemplate("delimited-block-paragraph", tmpls.DelimitedBlockParagraph, err)
 		r.documentDetails, err = r.newTemplate("document-details", tmpls.DocumentDetails, err)
 		r.documentAuthorDetails, err = r.newTemplate("document-author-details", tmpls.DocumentAuthorDetails, err)

pkg/renderer/sgml/templates.go

@@ -11,6 +11,7 @@ type Templates struct {
 	BoldText                  string
 	CalloutList               string
 	CalloutListItem           string
+	CalloutRef                string
 	DelimitedBlockParagraph   string
 	DocumentDetails           string
 	DocumentAuthorDetails     string

pkg/renderer/sgml/xhtml5/delimited_block_test.go

@@ -357,6 +357,36 @@ int main(int argc, char **argv);
 			Expect(RenderXHTML(source)).To(MatchHTML(expected))
 		})
 
+		It("with C content, and callout", func() {
+			source := `:source-highlighter: chroma
+[source, c]
+----
+#include <stdio.h>
+
+printf("Hello world!\n"); // <1>
+<a>link</a>
+----
+<1> A greeting
+`
+			expected := `<div class="listingblock">
+<div class="content">
+<pre class="chroma highlight"><code data-lang="c"><span class="tok-cp">#include</span> <span class="tok-cpf">&lt;stdio.h&gt;</span><span class="tok-cp">
+</span><span class="tok-cp"></span>
+<span class="tok-n">printf</span><span class="tok-p">(</span><span class="tok-s">&#34;Hello world!</span><span class="tok-se">\n</span><span class="tok-s">&#34;</span><span class="tok-p">);</span> <span class="tok-c1">// <b class="conum">(1)</b>
+</span><span class="tok-c1"></span><span class="tok-o">&lt;</span><span class="tok-n">a</span><span class="tok-o">&gt;</span><span class="tok-n">link</span><span class="tok-o">&lt;/</span><span class="tok-n">a</span><span class="tok-o">&gt;</span></code></pre>
+</div>
+</div>
+<div class="colist arabic">
+<ol>
+<li>
+<p>A greeting</p>
+</li>
+</ol>
+</div>
+`
+			Expect(RenderXHTML(source)).To(MatchHTML(expected))
+		})
+
 		It("with other content", func() {
 			source := `----
   a<<b