Merge pull request #242 from Witiko/fix/do-not-fold-spaces-in-unit-tests

Do not fold tabs and spaces into a single space token in unit tests

CHANGES.md

@@ -13,6 +13,10 @@ Default Renderer Prototypes:
 - Use `paralist` LaTeX package to define default renderer prototypes for
   fancy lists when `fancyList` Lua option is enabled. (#241)
 
+Unit Tests:
+
+- Do not fold tabs and spaces into a single space token. (#242)
+
 Speed Improvements:
 
 - Only make backticks special when `codeSpans` or `fencedCode` are enabled.

Makefile

@@ -15,7 +15,7 @@ ARCHIVES=$(TDSARCHIVE) $(CTANARCHIVE) $(DISTARCHIVE)
 EXAMPLES_RESOURCES=examples/example.md examples/scientists.csv
 EXAMPLES_SOURCES=examples/context-mkii.tex examples/context-mkiv.tex examples/latex.tex
 EXAMPLES=examples/context-mkii.pdf examples/context-mkiv.pdf \
-  examples/latex-pdftex.pdf examples/latex-luatex.pdf examples/latex-xetex.pdf \
+  examples/latex-pdftex.pdf examples/latex-luatex.pdf \
   examples/latex-tex4ht.html examples/latex-tex4ht.css
 TESTS=tests/test.sh tests/support/*.tex tests/templates/*/*.tex.m4 \
   tests/templates/*/COMMANDS.m4 tests/testfiles/*/*.test
@@ -74,7 +74,7 @@ $(GITHUB_PAGES): $(HTML_USER_MANUAL)
 
 # This target extracts the source files out of the DTX archive.
 $(EXTRACTABLES): $(INSTALLER) $(DTXARCHIVE)
-	xetex $<
+	luatex $<
 	sed -i \
 	    -e 's#(((VERSION)))#$(VERSION)#g' \
 	    -e 's#(((LASTMODIFIED)))#$(LASTMODIFIED)#g' \

README.md

@@ -165,9 +165,8 @@ For further information, consult one of the following:
 1. The user manual for either [the released version][manual-tex-live] or
    [the latest development version][manual-latest], which can be produced by
    interpreting the `markdown.ins` file using a Unicode-aware TeX engine, such
-   as XeTeX (`xetex markdown.ins`) or LuaTeX (`luatex markdown.ins`). The
-   manual will reside in the file `markdown.md` and the CSS stylesheet
-   `markdown.css`.
+   as LuaTeX (`luatex markdown.ins`). The manual will reside in the file
+   `markdown.md` and the CSS stylesheet `markdown.css`.
 2. The technical documentation for either [the released version][techdoc-tex-live]
    or [the latest development version][techdoc-latest], which can be typeset by
    running the [LaTeXMK][] tool on the `markdown.dtx` file (`latexmk
@@ -279,11 +278,11 @@ Some useful commands, such as building the release archives and typesetting
 the documentation, are placed in the `Makefile` file for ease of maintenance.
 
 When the file `markdown.ins` is interpreted using a Unicode-aware TeX engine,
-such as XeTeX (`xetex markdown.ins`) or LuaTeX (`luatex markdown.ins`), several
-files are produced from the `markdown.dtx` document. The `make base` command
-is provided by `Makefile` for convenience. In `markdown.dtx`, the boundaries
-between the produced files are marked up using an XML-like syntax provided by
-the [l3docstrip][] plain TeX package.
+such LuaTeX (`luatex markdown.ins`), several files are produced from the
+`markdown.dtx` document. The `make base` command is provided by `Makefile` for
+convenience. In `markdown.dtx`, the boundaries between the produced files are
+marked up using an XML-like syntax provided by the [l3docstrip][] plain TeX
+package.
 
 Running the [LaTeXMK][] tool on the `markdown.dtx` file
 (`latexmk markdown.dtx`) after the Markdown package has been

examples/Makefile

@@ -2,7 +2,7 @@
 
 AUXFILES=*.tmp *.tui *.tuo *.mp *.tuc *.markdown.in *.markdown.out \
   *.markdown.lua *.log *.aux *.dvi *.idv *.lg *.tmp *.xref *.4ct *.4tc \
-  latex-pdftex.tex latex-xetex.tex latex-luatex.tex latex-tex4ht.tex \
+  latex-pdftex.tex latex-luatex.tex latex-tex4ht.tex \
   example.tex *.debug-extensions.json debug-extensions.json
 AUXDIRS=_markdown_*/
 LUACLI_OPTIONS=\
@@ -24,7 +24,7 @@ LUACLI_OPTIONS=\
   tableCaptions=true \
   taskLists=true
 OUTPUT=context-mkii.pdf context-mkiv.pdf latex-pdftex.pdf \
-  latex-luatex.pdf latex-xetex.pdf latex-tex4ht.html latex-tex4ht.css
+  latex-luatex.pdf latex-tex4ht.html latex-tex4ht.css
 
 # This is the default pseudo-target.
 all: $(OUTPUT)
@@ -44,12 +44,6 @@ latex-pdftex.pdf: latex.tex example.tex
 	pdflatex --shell-escape latex-pdftex
 	pdflatex --shell-escape latex-pdftex
 
-# This target typesets the LaTeX example using the XeTeX engine.
-latex-xetex.pdf: latex.tex example.tex
-	cp latex.tex latex-xetex.tex
-	xelatex --shell-escape --8bit latex-xetex
-	xelatex --shell-escape --8bit latex-xetex
-
 # This target typesets the LaTeX example using the LuaTeX engine.
 latex-luatex.pdf: latex.tex example.tex
 	cp latex.tex latex-luatex.tex

markdown.dtx

@@ -20765,7 +20765,7 @@ function M.writer.new(options)
 % \end{markdown}
 %  \begin{macrocode}
   function self.pack(name)
-    return [[\input ]] .. name .. [[\relax]]
+    return [[\input{]] .. name .. [[}\relax]]
   end
 %    \end{macrocode}
 % \par

tests/support/markdownthemewitiko_markdown_test.sty

@@ -18,9 +18,13 @@
     bracketedSpanAttributeContextEnd = {%
       \TYPE{bracketedSpanAttributeContextEnd}},
     documentBegin = {%
+      \begingroup
+      \catcode"09=12%  Prevent spaces (U+0009) from folding into a single space token
+      \catcode"20=12%  Prevent tabs (U+0020) from folding into a single space token
       \TYPE{documentBegin}},
     documentEnd = {%
-      \TYPE{documentEnd}},
+      \TYPE{documentEnd}%
+      \endgroup},
     interblockSeparator = {%
       \TYPE{interblockSeparator}%
       \GOBBLE},

tests/support/plain-setup.tex

@@ -16,11 +16,13 @@
 \def\markdownRendererBracketedSpanAttributeContextEnd{%
   \TYPE{bracketedSpanAttributeContextEnd}}%
 \def\markdownRendererDocumentBegin{%
-  \TYPE{documentBegin}}%
-\def\markdownRendererDocumentBegin{%
+  \begingroup
+  \catcode"09=12%  Prevent spaces (U+0009) from folding into a single space token
+  \catcode"20=12%  Prevent tabs (U+0020) from folding into a single space token
   \TYPE{documentBegin}}%
 \def\markdownRendererDocumentEnd{%
-  \TYPE{documentEnd}}%
+  \TYPE{documentEnd}%
+  \endgroup}%
 \def\markdownRendererInterblockSeparator#1{%
   \TYPE{interblockSeparator}}%
 \def\markdownRendererLineBreak#1{%

tests/templates/latex/COMMANDS.m4

@@ -1,3 +1,2 @@
 pdflatex --shell-escape                  --interaction=nonstopmode  TEST_FILENAME
-xelatex  --shell-escape --8bit           --interaction=nonstopmode  TEST_FILENAME
 lualatex                                 --interaction=nonstopmode  TEST_FILENAME

tests/templates/plain/COMMANDS.m4

@@ -1,3 +1,2 @@
 pdftex   --shell-escape                  --interaction=nonstopmode  TEST_FILENAME
-xetex    --shell-escape --8bit           --interaction=nonstopmode  TEST_FILENAME
 luatex                                   --interaction=nonstopmode  TEST_FILENAME

tests/testfiles/lunamark-markdown/html.test

@@ -38,7 +38,7 @@ inputBlockHtmlElement: ./_markdown_test/db2433856814aefaa62794558a22a63c.verbati
 interblockSeparator
 emphasis: There is (inlineHtmlTag: <inline tag="value">)(inlineHtmlTag: </inline>) support.(inlineHtmlTag: <br/>)
 emphasis: There is (inlineHtmlComment: (emphasis: comment)) support.
-emphasis: There is support.
+emphasis: There is  support.
 interblockSeparator
 blockHtmlCommentBegin
 emphasis: block comment

tests/testfiles/lunamark-markdown/no-slice.test

@@ -168,7 +168,7 @@ END image
 interblockSeparator
 interblockSeparator
 interblockSeparator
-codeSpan: latex (backslash)documentclass(leftBrace)article(rightBrace) (backslash)begin(leftBrace)document(rightBrace) Hello world! (backslash)end(leftBrace)document(rightBrace) 
+codeSpan: latex (backslash)documentclass(leftBrace)article(rightBrace) (backslash)begin(leftBrace)document(rightBrace)   Hello world! (backslash)end(leftBrace)document(rightBrace) 
 interblockSeparator
 headerAttributeContextEnd
 headerAttributeContextBegin