switch to commonmark for uniformity and more extensions

Distribution/Server/Features/PackageCandidates.hs

@@ -33,23 +33,19 @@ import qualified Distribution.Server.Framework.ResponseContentTypes as Resource
 import Distribution.Server.Features.Security.Migration
 
 import Distribution.Server.Util.ServeTarball
+import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown)
 import Distribution.Server.Pages.Template (hackagePage)
 
 import Distribution.Text
 import Distribution.Package
 import Distribution.Version
 
-import qualified Cheapskate      as Markdown (markdown, Options(..))
-import qualified Cheapskate.Html as Markdown (renderDoc)
-import qualified Text.Blaze.Html.Renderer.Pretty as Blaze (renderHtml)
 import qualified Data.Text                as T
 import qualified Data.Text.Encoding       as T
 import qualified Data.Text.Encoding.Error as T
 import qualified Data.ByteString.Lazy as BS (ByteString, toStrict)
-import qualified Data.ByteString.Char8 as C8
 import qualified Text.XHtml.Strict as XHtml
 import           Text.XHtml.Strict ((<<), (!))
-import System.FilePath.Posix (takeExtension)
 import Data.Aeson (Value (..), object, toJSON, (.=))
 
 import Data.Function (fix)
@@ -589,7 +585,7 @@ candidatesFeature ServerEnv{serverBlobStore = store}
               [ XHtml.h2 << title
               , XHtml.thediv ! [XHtml.theclass "embedded-author-content"]
                             << if supposedToBeMarkdown filename
-                                 then renderMarkdown contents
+                                 then renderMarkdown filename contents
                                  else XHtml.thediv ! [XHtml.theclass "preformatted"]
                                                   << unpackUtf8 contents
               ]
@@ -608,25 +604,6 @@ candidatesFeature ServerEnv{serverBlobStore = store}
                        ["index.html"] (display (packageId pkg)) fp index
                        [Public, maxAgeMinutes 5] etag
 
-renderMarkdown :: BS.ByteString -> XHtml.Html
-renderMarkdown = XHtml.primHtml . Blaze.renderHtml
-               . Markdown.renderDoc . Markdown.markdown opts
-               . T.decodeUtf8With T.lenientDecode . convertNewLine . BS.toStrict
-  where
-    opts =
-      Markdown.Options
-        { Markdown.sanitize           = True
-        , Markdown.allowRawHtml       = False
-        , Markdown.preserveHardBreaks = False
-        , Markdown.debug              = False
-        }
-
-convertNewLine :: C8.ByteString -> C8.ByteString
-convertNewLine = C8.filter (/= '\r')
-
-supposedToBeMarkdown :: FilePath -> Bool
-supposedToBeMarkdown fname = takeExtension fname `elem` [".md", ".markdown"]
-
 unpackUtf8 :: BS.ByteString -> String
 unpackUtf8 = T.unpack
            . T.decodeUtf8With T.lenientDecode

Distribution/Server/Features/PackageContents.hs

@@ -17,24 +17,20 @@ import Distribution.Server.Packages.Types
 import Distribution.Server.Packages.Render
 import Distribution.Server.Features.Users
 import Distribution.Server.Util.ServeTarball
+import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown)
 import Distribution.Server.Pages.Template (hackagePage)
 
 import Distribution.Text
 import Distribution.Package
 import Distribution.PackageDescription
 
-import qualified Cheapskate      as Markdown (markdown, Options(..))
-import qualified Cheapskate.Html as Markdown (renderDoc)
-import qualified Text.Blaze.Html.Renderer.Pretty as Blaze (renderHtml)
 import qualified Data.Text                as T
 import qualified Data.Text.Encoding       as T
 import qualified Data.Text.Encoding.Error as T
 import qualified Data.ByteString.Lazy as BS (ByteString, toStrict)
-import qualified Data.ByteString.Char8 as C8
 import qualified Text.XHtml.Strict as XHtml
 import qualified Distribution.Utils.ShortText as ST
 import           Text.XHtml.Strict ((<<), (!))
-import System.FilePath.Posix (takeExtension)
 
 
 data PackageContentsFeature = PackageContentsFeature {
@@ -162,7 +158,7 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{
                  [ XHtml.h2 << title2
                  , XHtml.thediv ! [XHtml.theclass "embedded-author-content"]
                                << if supposedToBeMarkdown filename
-                                    then renderMarkdown contents
+                                    then renderMarkdown filename contents
                                     else XHtml.thediv ! [XHtml.theclass "preformatted"]
                                                      << unpackUtf8 contents
                  ]
@@ -196,7 +192,7 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{
               [ XHtml.h2 << title
               , XHtml.thediv ! [XHtml.theclass "embedded-author-content"]
                             << if supposedToBeMarkdown filename
-                                 then renderMarkdown contents
+                                 then renderMarkdown filename contents
                                  else XHtml.thediv ! [XHtml.theclass "preformatted"]
                                                   << unpackUtf8 contents
               ]
@@ -214,25 +210,6 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{
                        [] (display (packageId pkg)) fp index
                        [Public, maxAgeDays 30] etag
 
-renderMarkdown :: BS.ByteString -> XHtml.Html
-renderMarkdown = XHtml.primHtml . Blaze.renderHtml
-               . Markdown.renderDoc . Markdown.markdown opts
-               . T.decodeUtf8With T.lenientDecode . convertNewLine . BS.toStrict
-  where
-    opts =
-      Markdown.Options
-        { Markdown.sanitize           = True
-        , Markdown.allowRawHtml       = False
-        , Markdown.preserveHardBreaks = False
-        , Markdown.debug              = False
-        }
-
-convertNewLine :: C8.ByteString -> C8.ByteString
-convertNewLine = C8.filter (/= '\r')
-
-supposedToBeMarkdown :: FilePath -> Bool
-supposedToBeMarkdown fname = takeExtension fname `elem` [".md", ".markdown"]
-
 unpackUtf8 :: BS.ByteString -> String
 unpackUtf8 = T.unpack
            . T.decodeUtf8With T.lenientDecode

Distribution/Server/Features/PackageFeed.hs

@@ -11,17 +11,15 @@ import Distribution.Server.Packages.Types
 import qualified Distribution.Server.Users.Users as Users
 import Distribution.Server.Users.Users (Users)
 import Distribution.Server.Util.ServeTarball (loadTarEntry)
+import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown)
 import Distribution.Server.Pages.Package () -- for ShortText html instance, for now.
 
 import Distribution.Package
 import Distribution.PackageDescription
 import Distribution.Text
 import Distribution.Utils.ShortText (fromShortText)
 
-import qualified Cheapskate as Markdown (markdown, Options(..))
-import qualified Cheapskate.Html as Markdown (renderDoc)
 import qualified Data.ByteString.Lazy as BS (ByteString, toStrict)
-import qualified Data.ByteString.Char8 as C8
 import Data.List (sortOn)
 import Data.Maybe (listToMaybe)
 import Data.Ord (Down(..))
@@ -31,8 +29,6 @@ import qualified Data.Text.Encoding.Error as T
 import Data.Time.Clock (UTCTime, getCurrentTime)
 import Data.Time.Format
 import Network.URI( URI(..), uriToString )
-import System.FilePath.Posix (takeExtension)
-import qualified Text.Blaze.Html.Renderer.Pretty as Blaze (renderHtml)
 import qualified Text.RSS as RSS
 import Text.RSS ( RSS(RSS) )
 import qualified Text.XHtml.Strict as XHtml
@@ -99,7 +95,7 @@ packageFeedFeature ServerEnv{..}
             Left _ -> return (pkg, XHtml.primHtml "(No changelog found.)")
             Right (_, content) ->
               if supposedToBeMarkdown filename
-                then return (pkg, renderMarkdown content)
+                then return (pkg, renderMarkdown filename content)
                 else return (pkg, XHtml.pre << unpackUtf8 content)
 
 renderPackageFeed :: Users -> URI -> UTCTime -> PackageName -> [(PkgInfo, XHtml.Html)] -> RSS
@@ -144,24 +140,6 @@ feedItems users hostURI (pkgInfo, chlog) =
         pd = packageDescription (pkgDesc pkgInfo)
         d dt dd = XHtml.dterm (XHtml.toHtml dt) +++ XHtml.ddef (XHtml.toHtml dd)
 
-renderMarkdown :: BS.ByteString -> XHtml.Html
-renderMarkdown = XHtml.primHtml . Blaze.renderHtml
-               . Markdown.renderDoc . Markdown.markdown opts
-               . T.decodeUtf8With T.lenientDecode . convertNewLine . BS.toStrict
-  where
-    opts =
-      Markdown.Options
-        { Markdown.sanitize           = True
-        , Markdown.allowRawHtml       = False
-        , Markdown.preserveHardBreaks = False
-        , Markdown.debug              = False
-        }
-
-convertNewLine :: C8.ByteString -> C8.ByteString
-convertNewLine = C8.filter (/= '\r')
-
-supposedToBeMarkdown :: FilePath -> Bool
-supposedToBeMarkdown fname = takeExtension fname `elem` [".md", ".markdown"]
 
 unpackUtf8 :: BS.ByteString -> String
 unpackUtf8 = T.unpack

Distribution/Server/Pages/Package.hs

@@ -20,7 +20,6 @@ module Distribution.Server.Pages.Package
   , maintainerSection
   , downloadSection
   , moduleToDocUrl
-  , renderMarkdown
   ) where
 
 import Distribution.Server.Features.PreferredVersions
@@ -31,6 +30,7 @@ import Distribution.Server.Pages.Package.HaddockHtml
 import Distribution.Server.Packages.ModuleForest
 import Distribution.Server.Packages.Render
 import Distribution.Server.Users.Types (userStatus, userName, isActiveAccount)
+import Distribution.Server.Util.Markdown (renderMarkdownRel, supposedToBeMarkdown)
 import Data.TarIndex (TarIndex)
 import qualified Data.TarIndex as Tar
 
@@ -52,15 +52,9 @@ import Control.Arrow            (second)
 import System.FilePath.Posix    ((</>), (<.>), takeFileName)
 import Data.Time.Locale.Compat  (defaultTimeLocale)
 import Data.Time.Format         (formatTime)
-import System.FilePath.Posix    (takeExtension)
-import Network.URI              (isRelativeReference)
-
-import qualified Cheapskate      as Markdown (markdown, Options(..), Inline(Link), walk)
-import qualified Cheapskate.Html as Markdown (renderDoc)
 
 import qualified Documentation.Haddock.Markup as Haddock
 
-import qualified Text.Blaze.Html.Renderer.Pretty as Blaze (renderHtml)
 import qualified Data.Text                as T
 import qualified Data.Text.Encoding       as T
 import qualified Data.Text.Encoding.Error as T (lenientDecode)
@@ -174,37 +168,12 @@ readmeSection PackageRender { rendReadme = Just (_, _etag, _, filename)
     , toHtml "]"
     , thediv ! [theclass "embedded-author-content"]
             << if supposedToBeMarkdown filename
-                 then renderMarkdown (T.pack name) content
+                 then renderMarkdownRel name content
                  else pre << unpackUtf8 content
     ] where
       name = display pkgid
 readmeSection _ _ = []
 
-updateRelativeLinks :: T.Text -> Markdown.Inline -> Markdown.Inline
-updateRelativeLinks name (Markdown.Link inls url title) =
-  Markdown.Link inls url' title
-  where url' = if isRelativeReference $ T.unpack url then name <> T.pack "/src" <> url else url
-updateRelativeLinks _ x = x
-
-renderMarkdown :: T.Text -> BS.ByteString -> Html
-renderMarkdown name = primHtml . Blaze.renderHtml
-               . Markdown.renderDoc . Markdown.walk (updateRelativeLinks name)
-               . Markdown.markdown opts
-                 -- workaround for https://github.com/jgm/cheapskate/issues/25
-               . T.replace (T.pack "\r\n") (T.pack "\n")
-               . T.decodeUtf8With T.lenientDecode
-               . BS.toStrict
-  where
-    opts =
-      Markdown.Options
-        { Markdown.sanitize           = True
-        , Markdown.allowRawHtml       = False
-        , Markdown.preserveHardBreaks = False
-        , Markdown.debug              = False
-        }
-
-supposedToBeMarkdown :: FilePath -> Bool
-supposedToBeMarkdown fname = takeExtension fname `elem` [".md", ".markdown"]
 
 unpackUtf8 :: BS.ByteString -> String
 unpackUtf8 = T.unpack

Distribution/Server/Pages/PackageFromTemplate.hs

@@ -18,6 +18,7 @@ import Distribution.Server.Packages.PackageIndex (PackageIndex)
 import Distribution.Server.Packages.Types
 import Distribution.Server.Features.PackageCandidates
 import Distribution.Server.Users.Types (userStatus, userName, isActiveAccount)
+import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown)
 import Data.TarIndex (TarIndex)
 import Distribution.Server.Features.Distro.Types
 
@@ -35,7 +36,6 @@ import Data.List                (intersperse)
 import System.FilePath.Posix    ((</>), takeFileName, dropTrailingPathSeparator)
 import Data.Time.Locale.Compat  (defaultTimeLocale)
 import Data.Time.Format         (formatTime)
-import System.FilePath.Posix    (takeExtension)
 
 import qualified Data.Text                as T
 import qualified Data.Text.Encoding       as T
@@ -491,13 +491,11 @@ readmeSection PackageRender { rendReadme = Just (_, _etag, _, filename), rendPkg
               (Just content) =
     [ thediv ! [theclass "embedded-author-content"]
             << if supposedToBeMarkdown filename
-                 then Old.renderMarkdown (T.pack $ display pkgid) content
+                 then renderMarkdown (display pkgid) content
                  else pre << unpackUtf8 content
     ]
 readmeSection _ _ = []
 
-supposedToBeMarkdown :: FilePath -> Bool
-supposedToBeMarkdown fname = takeExtension fname `elem` [".md", ".markdown"]
 
 unpackUtf8 :: BS.ByteString -> String
 unpackUtf8 = T.unpack

Distribution/Server/Pages/Template.hs

@@ -36,6 +36,7 @@ hackagePageWith headExtra docTitle docSubtitle docContent bodyExtra =
                             , thetype "image/png"] << noHtml
                 , meta ! [ name "viewport"
                          , content "width=device-width, initial-scale=1"]
+                , (script noHtml) ! [ src "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js", thetype "text/javascript"]
                 -- if Search is enabled
                 , thelink ! [ rel "search", href "/packages/opensearch.xml"
                             , thetype "application/opensearchdescription+xml"