refactor(parser/renderer): special characters at parser level (#734)

Parsing special characters (`>`, `<`, `&`) as well as
`+` for output compatibility with Asciidoctor

These special characters are stored in the new `SpecialCharacter`
type and can be converted into HTML entities by the HTML5
and XHTML5 backends. 
Other backends may decide to display the values as-is.

This removes the usage of HTML entities during
attribute substitutions at the parser level, which could 
cause troubles for future, non-HTML backends. 

Also, it removes the need to the `sanitized` type 
(an alias for `html.HTML`)

Fixes #729

Signed-off-by: Xavier Coulon <[email protected]>

LIMITATIONS.adoc

@@ -28,7 +28,7 @@ Libasciidoc does not emit one.
 
 == Passthroughs
 
-The Inline pass macro will not render nothing at all instead of an empty paragraph
+The inline passthrough macro will not render nothing at all instead of an empty paragraph
 when the macro is empty. I.e.:
 
 ....
@@ -43,6 +43,31 @@ will produce no HTML element at all, whereas Asciidoc/Asciidoctor will produce :
 </div>
 ....
 
+Also, the inline passthrough macro does not support line breaks within its content. I.e:
+
+....
++hello,
+world+
+....
+
+will produce:
+
+....
+<div class="paragraph">
+<p>&#43;hello,
+world&#43;</p>
+</div>
+....
+
+whereas Asciidoctor will produce :
+
+....
+<div class="paragraph">
+<p>hello,
+world</p>
+</div>
+....
+
 == Constrained Monospace Text and Quoted Strings or Apostrophes
 
 Constrained or imbalanced monospace text may not act fully constrained, or may be confused in the presence of

go.mod

@@ -20,7 +20,6 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.6.1
-	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect

go.sum

@@ -159,7 +159,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -173,8 +172,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -186,10 +183,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
@@ -198,7 +192,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -227,8 +220,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190830223141-573d9926052a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 h1:/e4fNMHdLn7SQSxTrRZTma2xjQW6ELdxcnpqMhpo9X4=
-golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6 h1:nULzSsKgihxFGLnQFv2T7lE5vIhOtg8ZPpJHapEt7o0=
 golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=

libasciidoc.go

@@ -4,11 +4,12 @@ package libasciidoc
 
 import (
 	"fmt"
-	"github.com/bytesparadise/libasciidoc/pkg/renderer/sgml/xhtml5"
 	"io"
 	"os"
 	"time"
 
+	"github.com/bytesparadise/libasciidoc/pkg/renderer/sgml/xhtml5"
+
 	"github.com/bytesparadise/libasciidoc/pkg/configuration"
 	"github.com/bytesparadise/libasciidoc/pkg/parser"
 	"github.com/bytesparadise/libasciidoc/pkg/renderer"

pkg/parser/delimited_block_test.go

@@ -644,8 +644,17 @@ import <a>
 						types.Paragraph{
 							Lines: []interface{}{
 								[]interface{}{
+									types.SpecialCharacter{
+										Content: "<",
+									},
+									types.StringElement{
+										Content: "a",
+									},
+									types.SpecialCharacter{
+										Content: ">",
+									},
 									types.StringElement{
-										Content: "<a> an import",
+										Content: " an import",
 									},
 								},
 							},
@@ -2544,8 +2553,17 @@ import <a>
 						types.Paragraph{
 							Lines: []interface{}{
 								[]interface{}{
+									types.SpecialCharacter{
+										Content: "<",
+									},
+									types.StringElement{
+										Content: "a",
+									},
+									types.SpecialCharacter{
+										Content: ">",
+									},
 									types.StringElement{
-										Content: "<a> an import",
+										Content: " an import",
 									},
 								},
 							},

pkg/parser/document_attributes_test.go

@@ -431,7 +431,7 @@ John  Foo Doe  <[email protected]>; Jane the_Doe <[email protected]>`
 									Email:    "[email protected]",
 								},
 								{
-									FullName: "Jane the Doe", // fill name was sanitized
+									FullName: "Jane the Doe",
 									Email:    "[email protected]",
 								},
 							},

pkg/parser/labeled_list_test.go

@@ -1813,7 +1813,7 @@ level 2::: description 2`
 									types.Paragraph{
 										Lines: []interface{}{
 											[]interface{}{
-												types.StringElement{Content: "&"},
+												types.SpecialCharacter{Content: "&"},
 											},
 										},
 									},

pkg/parser/ordered_list_test.go

@@ -1048,7 +1048,7 @@ print("one")
 										types.Paragraph{
 											Lines: []interface{}{
 												[]interface{}{
-													types.StringElement{Content: "&"},
+													types.SpecialCharacter{Content: "&"},
 												},
 											},
 										},

pkg/parser/paragraph_test.go

@@ -156,7 +156,9 @@ foo`
 						types.Paragraph{
 							Lines: []interface{}{
 								[]interface{}{
-									types.StringElement{Content: "C++"},
+									types.StringElement{Content: "C"},
+									types.SpecialCharacter{Content: "+"},
+									types.SpecialCharacter{Content: "+"},
 								},
 								[]interface{}{
 									types.StringElement{Content: "foo"},
@@ -804,7 +806,9 @@ a paragraph`
 						types.Paragraph{
 							Lines: []interface{}{
 								[]interface{}{
-									types.StringElement{Content: "hello + world"},
+									types.StringElement{Content: "hello "},
+									types.SpecialCharacter{Content: "+"},
+									types.StringElement{Content: " world"},
 								},
 							},
 						},