Text/status parsing fixes

CONTRIBUTING.md

@@ -139,7 +139,7 @@ We use [golangci-lint](https://golangci-lint.run/) for linting. To run this loca
 Then, you can run the linter with:
 
 ```bash
-golangci-lint run
+golangci-lint run --tests=false
 ```
 
 Note that this linter also runs as a job on the Github repo, so if you make a PR that doesn't pass the linter, it will be rejected. As such, it's good practice to run the linter locally before pushing or opening a PR.
@@ -155,7 +155,7 @@ go get -u github.com/golang/lint/golint
 To run the linter, use:
 
 ```bash
-golint ./...
+golint ./internal/...
 ```
 
 Then make sure to run `go fmt ./...` to update whitespace and other opinionated formatting.

README.md

@@ -143,6 +143,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude 
 * [superseriousbusiness/exifremove](https://github.com/superseriousbusiness/exifremove) forked from [scottleedavis/go-exif-remove](https://github.com/scottleedavis/go-exif-remove); EXIF data removal. [MIT License](https://spdx.org/licenses/MIT.html).
 * [superseriousbusiness/oauth2](https://github.com/superseriousbusiness/oauth2) forked from [go-oauth2/oauth2](https://github.com/go-oauth2/oauth2); oauth server framework and token handling. [MIT License](https://spdx.org/licenses/MIT.html).
 * [go-swagger/go-swagger](https://github.com/go-swagger/go-swagger); Swagger OpenAPI spec generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
+* [tdewolff/minify](https://github.com/tdewolff/minify); HTML minification. [MIT License](https://spdx.org/licenses/MIT.html).
 * [urfave/cli](https://github.com/urfave/cli); command-line interface framework. [MIT License](https://spdx.org/licenses/MIT.html).
 * [wagslane/go-password-validator](https://github.com/wagslane/go-password-validator); password strength validation. [MIT License](https://spdx.org/licenses/MIT.html).
 

dockerbuild.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker build -t "superseriousbusiness/gotosocial:$(cat version)" .
+docker build -t "superseriousbusiness/gotosocial:$(git rev-parse --abbrev-ref HEAD)" .

dockerpush.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker push "superseriousbusiness/gotosocial:$(cat version)"
+docker push "superseriousbusiness/gotosocial:$(git rev-parse --abbrev-ref HEAD)"

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203
 	github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB
+	github.com/tdewolff/minify/v2 v2.9.21
 	github.com/tidwall/buntdb v1.2.4 // indirect
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect

go.sum

@@ -46,6 +46,7 @@ github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414
 github.com/buckket/go-blurhash v1.1.0 h1:X5M6r0LIvwdvKiUtiNcRL2YlmOfMzYobI3VCKCZc9Do=
 github.com/buckket/go-blurhash v1.1.0/go.mod h1:aT2iqo5W9vu9GpyoLErKfTHwgODsZp3bQfXjXJUxNb8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -92,6 +93,7 @@ github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3
 github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e h1:ojqYA1mU6LuRm8XzrVOvyfb000y59cbUcu6Wt8sFSAs=
 github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e/go.mod h1:KVK+/Hul09ujXAGq+42UBgCTnXkiJZRnLYdURGjQUwo=
 github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -266,6 +268,7 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
+github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -323,6 +326,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -335,6 +339,12 @@ github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 h1
 github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203/go.mod h1:0Xw5cYMOYpgaWs+OOSx41ugycl2qvKTi9tlMMcZhFyY=
 github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB h1:dzMVC+oPTxFL5cv29egBrftlqIWPXQ6/VzkuoySwgm4=
 github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB/go.mod h1:8p0a/BEN9hhsGzE3tPaFFlIZgxAaLyLN5KY0bPg9ZBc=
+github.com/tdewolff/minify/v2 v2.9.21 h1:nO4s1PEMy7aRjlIlbr3Jgr+bJby8QYuifa2Vs2f9lh4=
+github.com/tdewolff/minify/v2 v2.9.21/go.mod h1:PoDBts2L7sCwUT28vTAlozGeD6qxjrrihtin4bR/RMM=
+github.com/tdewolff/parse/v2 v2.5.19 h1:Kjaj3KQOx/4elIxlBSglus4E2oMfdROphvbq2b+OBZ0=
+github.com/tdewolff/parse/v2 v2.5.19/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
+github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
+github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
 github.com/tidwall/btree v0.5.0 h1:IBfCtOj4uOMQcodv3wzYVo0zPqSJObm71mE039/dlXY=
 github.com/tidwall/btree v0.5.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=

internal/api/client/status/statuscreate_test.go

@@ -165,7 +165,7 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
 	err = json.Unmarshal(b, statusReply)
 	assert.NoError(suite.T(), err)
 
-	assert.Equal(suite.T(), "\u003cp\u003e\u003ca href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003etest\u003c/span\u003e\u003c/a\u003e alright, should be able to post \u003ca href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003elinks\u003c/span\u003e\u003c/a\u003e with fragments in them now, let\u0026#39;s see........\u003cbr/\u003e\u003cbr/\u003e\u003ca href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\"\u003edocs.gotosocial.org/en/latest/user_guide/posts/#links\u003c/a\u003e\u003cbr/\u003e\u003cbr/\u003e\u003ca href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003egotosocial\u003c/span\u003e\u003c/a\u003e\u003cbr/\u003e\u003cbr/\u003e(tobi remember to pull the docker image challenge)\u003c/p\u003e", statusReply.Content)
+	assert.Equal(suite.T(), "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
 }
 
 func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
@@ -198,7 +198,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
 	assert.NoError(suite.T(), err)
 
 	assert.Equal(suite.T(), "", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: <br/> here&#39;s an emoji that isn&#39;t in the db: :test_emoji:</p>", statusReply.Content)
+	assert.Equal(suite.T(), "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", statusReply.Content)
 
 	assert.Len(suite.T(), statusReply.Emojis, 1)
 	mastoEmoji := statusReply.Emojis[0]
@@ -314,7 +314,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
 	assert.NoError(suite.T(), err)
 
 	assert.Equal(suite.T(), "", statusResponse.SpoilerText)
-	assert.Equal(suite.T(), "<p>here&#39;s an image attachment</p>", statusResponse.Content)
+	assert.Equal(suite.T(), "<p>here's an image attachment</p>", statusResponse.Content)
 	assert.False(suite.T(), statusResponse.Sensitive)
 	assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility)
 

internal/processing/status/util_test.go

@@ -17,16 +17,16 @@ const statusText1 = `Another test @[email protected]
 #Hashtag
 
 Text`
-const statusText1ExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
-const statusText1ExpectedPartial = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/>#Hashtag<br/><br/>Text</p>`
+const statusText1ExpectedFull = "<p>Another test <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span><br><br><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a><br><br>Text</p>"
+const statusText1ExpectedPartial = "<p>Another test <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span><br><br>#Hashtag<br><br>Text</p>"
 
 const statusText2 = `Another test @[email protected]
 
 #Hashtag
 
 #hashTAG`
 
-const status2TextExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>hashTAG</span></a></p>`
+const status2TextExpectedFull = "<p>Another test <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span><br><br><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a><br><br><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashTAG</span></a></p>"
 
 type UtilTestSuite struct {
 	StatusStandardTestSuite

internal/text/common.go

@@ -20,6 +20,7 @@ package text
 
 import (
 	"fmt"
+	"html"
 	"strings"
 
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -29,23 +30,33 @@ import (
 // preformat contains some common logic for making a string ready for formatting, which should be used for all user-input text.
 func preformat(in string) string {
 	// do some preformatting of the text
-	// 1. Trim all the whitespace
-	s := strings.TrimSpace(in)
+
+	// 1. unescape everything that might be html escaped
+	s := html.UnescapeString(in)
+
+	// 2. trim leading or trailing whitespace
+	s = strings.TrimSpace(s)
 	return s
 }
 
 // postformat contains some common logic for html sanitization of text, wrapping elements, and trimming newlines and whitespace
 func postformat(in string) string {
 	// do some postformatting of the text
-	// 1. sanitize html to remove any dodgy scripts or other disallowed elements
-	s := SanitizeOutgoing(in)
-	// 2. wrap the whole thing in a paragraph
-	s = fmt.Sprintf(`<p>%s</p>`, s)
-	// 3. remove any cheeky newlines
-	s = strings.ReplaceAll(s, "\n", "")
-	// 4. remove any whitespace added as a result of the formatting
-	s = strings.TrimSpace(s)
-	return s
+
+	// 1. sanitize html to remove potentially dangerous elements
+	s := SanitizeHTML(in)
+
+	// 2. the sanitize step tends to escape characters inside codeblocks, which is behavior we don't want, so unescape everything again
+	s = html.UnescapeString(s)
+
+	// 3. minify html to remove any trailing newlines, spaces, unnecessary elements, etc etc
+	mini, err := minifyHTML(s)
+	if err != nil {
+		// if the minify failed, just return what we have
+		return s
+	}
+	// return minified version of the html
+	return mini
 }
 
 func (f *formatter) ReplaceTags(in string, tags []*gtsmodel.Tag) string {

internal/text/markdown.go

@@ -23,21 +23,14 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-var bfExtensions = blackfriday.NoIntraEmphasis |
-	blackfriday.FencedCode |
-	blackfriday.Autolink |
-	blackfriday.Strikethrough |
-	blackfriday.SpaceHeadings |
-	blackfriday.BackslashLineBreak
-
 func (f *formatter) FromMarkdown(md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
 	content := preformat(md)
 
 	// do the markdown parsing *first*
-	content = string(blackfriday.Run([]byte(content), blackfriday.WithExtensions(bfExtensions)))
+	contentBytes := blackfriday.Run([]byte(content))
 
 	// format tags nicely
-	content = f.ReplaceTags(content, tags)
+	content = f.ReplaceTags(string(contentBytes), tags)
 
 	// format mentions nicely
 	content = f.ReplaceMentions(content, mentions)

internal/text/markdown_test.go

@@ -0,0 +1,116 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors [email protected]
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package text_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/text"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+	simpleMarkdown = `# Title
+
+Here's a simple text in markdown.
+
+Here's a [link](https://example.org).`
+
+	simpleMarkdownExpected = "<h1>Title</h1><p>Here’s a simple text in markdown.</p><p>Here’s a <a href=\"https://example.org\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">link</a>.</p>"
+
+	withCodeBlockExpected = "<h1>Title</h1><p>Below is some JSON.</p><pre><code class=\"language-json\">{\n  \"key\": \"value\",\n  \"another_key\": [\n    \"value1\",\n    \"value2\"\n  ]\n}\n</code></pre><p>that was some JSON :)</p>"
+
+	withHashtag         = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
+	withHashtagExpected = "<h1>Title</h1><p>here’s a simple status that uses hashtag <a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a>!</p>"
+)
+
+var (
+	withCodeBlock = `# Title
+
+Below is some JSON.
+
+` + "```" + `json
+{
+  "key": "value",
+  "another_key": [
+    "value1",
+    "value2"
+  ]
+}
+` + "```" + `
+
+that was some JSON :)
+`
+)
+
+type MarkdownTestSuite struct {
+	TextStandardTestSuite
+}
+
+func (suite *MarkdownTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testTags = testrig.NewTestTags()
+	suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *MarkdownTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log)
+
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
+}
+
+func (suite *MarkdownTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *MarkdownTestSuite) TestParseSimple() {
+	s := suite.formatter.FromMarkdown(simpleMarkdown, nil, nil)
+	suite.Equal(simpleMarkdownExpected, s)
+}
+
+func (suite *MarkdownTestSuite) TestParseWithCodeBlock() {
+	fmt.Println(withCodeBlock)
+	s := suite.formatter.FromMarkdown(withCodeBlock, nil, nil)
+	suite.Equal(withCodeBlockExpected, s)
+}
+
+func (suite *MarkdownTestSuite) TestParseWithHashtag() {
+	foundTags := []*gtsmodel.Tag{
+		suite.testTags["Hashtag"],
+	}
+
+	s := suite.formatter.FromMarkdown(withHashtag, nil, foundTags)
+	suite.Equal(withHashtagExpected, s)
+}
+
+func TestMarkdownTestSuite(t *testing.T) {
+	suite.Run(t, new(MarkdownTestSuite))
+}