Link hashtag bug (#121)

* link + hashtag bug

* remove printlns

* tidy up some duplicated code

internal/api/client/status/statuscreate_test.go

@@ -57,6 +57,7 @@ func (suite *StatusCreateTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
+	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
@@ -69,6 +70,14 @@ func (suite *StatusCreateTestSuite) TearDownTest() {
 	testrig.StandardStorageTeardown(suite.storage)
 }
 
+var statusWithLinksAndTags = `#test alright, should be able to post #links with fragments in them now, let's see........
+
+https://docs.gotosocial.org/en/latest/user_guide/posts/#links
+
+#gotosocial
+
+(tobi remember to pull the docker image challenge)`
+
 // Post a new status with some custom visibility settings
 func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 
@@ -109,7 +118,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	assert.NoError(suite.T(), err)
 
 	assert.Equal(suite.T(), "hello hello", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "this is a brand new status! #helloworld", statusReply.Content)
+	assert.Equal(suite.T(), "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", statusReply.Content)
 	assert.True(suite.T(), statusReply.Sensitive)
 	assert.Equal(suite.T(), model.VisibilityPrivate, statusReply.Visibility)
 	assert.Len(suite.T(), statusReply.Tags, 1)
@@ -124,6 +133,43 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	assert.Equal(suite.T(), statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
 }
 
+func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
+
+	t := suite.testTokens["local_account_1"]
+	oauthToken := oauth.TokenToOauthToken(t)
+
+	// setup
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
+	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
+	ctx.Request.Form = url.Values{
+		"status": {statusWithLinksAndTags},
+	}
+	suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+	// check response
+
+	// 1. we should have OK from our call to the function
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+
+	fmt.Println(string(b))
+
+	statusReply := &model.Status{}
+	err = json.Unmarshal(b, statusReply)
+	assert.NoError(suite.T(), err)
+
+	assert.Equal(suite.T(), "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let&#39;s see........<br/><br/><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br/><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br/><br/>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
+}
+
 func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
 
 	t := suite.testTokens["local_account_1"]
@@ -154,7 +200,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
 	assert.NoError(suite.T(), err)
 
 	assert.Equal(suite.T(), "", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: ", statusReply.Content)
+	assert.Equal(suite.T(), "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: <br/> here&#39;s an emoji that isn&#39;t in the db: :test_emoji:</p>", statusReply.Content)
 
 	assert.Len(suite.T(), statusReply.Emojis, 1)
 	mastoEmoji := statusReply.Emojis[0]
@@ -228,7 +274,7 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() {
 	assert.NoError(suite.T(), err)
 
 	assert.Equal(suite.T(), "", statusReply.SpoilerText)
-	assert.Equal(suite.T(), fmt.Sprintf("hello @%s this reply should work!", testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
+	assert.Equal(suite.T(), fmt.Sprintf("<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@%s\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>%s</span></a></span> this reply should work!</p>", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
 	assert.False(suite.T(), statusReply.Sensitive)
 	assert.Equal(suite.T(), model.VisibilityPublic, statusReply.Visibility)
 	assert.Equal(suite.T(), testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID)
@@ -241,6 +287,8 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
 	t := suite.testTokens["local_account_1"]
 	oauthToken := oauth.TokenToOauthToken(t)
 
+	attachment := suite.testAttachments["local_account_1_unattached_1"]
+
 	// setup
 	recorder := httptest.NewRecorder()
 	ctx, _ := gin.CreateTestContext(recorder)
@@ -251,7 +299,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
 	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
 	ctx.Request.Form = url.Values{
 		"status":    {"here's an image attachment"},
-		"media_ids": {"7a3b9f77-ab30-461e-bdd8-e64bd1db3008"},
+		"media_ids": {attachment.ID},
 	}
 	suite.statusModule.StatusCreatePOSTHandler(ctx)
 
@@ -263,34 +311,32 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
 	b, err := ioutil.ReadAll(result.Body)
 	assert.NoError(suite.T(), err)
 
-	fmt.Println(string(b))
-
-	statusReply := &model.Status{}
-	err = json.Unmarshal(b, statusReply)
+	statusResponse := &model.Status{}
+	err = json.Unmarshal(b, statusResponse)
 	assert.NoError(suite.T(), err)
 
-	assert.Equal(suite.T(), "", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "here's an image attachment", statusReply.Content)
-	assert.False(suite.T(), statusReply.Sensitive)
-	assert.Equal(suite.T(), model.VisibilityPublic, statusReply.Visibility)
+	assert.Equal(suite.T(), "", statusResponse.SpoilerText)
+	assert.Equal(suite.T(), "<p>here&#39;s an image attachment</p>", statusResponse.Content)
+	assert.False(suite.T(), statusResponse.Sensitive)
+	assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility)
 
 	// there should be one media attachment
-	assert.Len(suite.T(), statusReply.MediaAttachments, 1)
+	assert.Len(suite.T(), statusResponse.MediaAttachments, 1)
 
 	// get the updated media attachment from the database
 	gtsAttachment := &gtsmodel.MediaAttachment{}
-	err = suite.db.GetByID(statusReply.MediaAttachments[0].ID, gtsAttachment)
+	err = suite.db.GetByID(statusResponse.MediaAttachments[0].ID, gtsAttachment)
 	assert.NoError(suite.T(), err)
 
 	// convert it to a masto attachment
 	gtsAttachmentAsMasto, err := suite.tc.AttachmentToMasto(gtsAttachment)
 	assert.NoError(suite.T(), err)
 
 	// compare it with what we have now
-	assert.EqualValues(suite.T(), statusReply.MediaAttachments[0], gtsAttachmentAsMasto)
+	assert.EqualValues(suite.T(), statusResponse.MediaAttachments[0], gtsAttachmentAsMasto)
 
 	// the status id of the attachment should now be set to the id of the status we just created
-	assert.Equal(suite.T(), statusReply.ID, gtsAttachment.StatusID)
+	assert.Equal(suite.T(), statusResponse.ID, gtsAttachment.StatusID)
 }
 
 func TestStatusCreateTestSuite(t *testing.T) {

internal/text/common.go

@@ -21,6 +21,9 @@ package text
 import (
 	"fmt"
 	"strings"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // preformat contains some common logic for making a string ready for formatting, which should be used for all user-input text.
@@ -35,7 +38,7 @@ func preformat(in string) string {
 func postformat(in string) string {
 	// do some postformatting of the text
 	// 1. sanitize html to remove any dodgy scripts or other disallowed elements
-	s := SanitizeHTML(in)
+	s := SanitizeOutgoing(in)
 	// 2. wrap the whole thing in a paragraph
 	s = fmt.Sprintf(`<p>%s</p>`, s)
 	// 3. remove any cheeky newlines
@@ -44,3 +47,29 @@ func postformat(in string) string {
 	s = strings.TrimSpace(s)
 	return s
 }
+
+func (f *formatter) ReplaceTags(in string, tags []*gtsmodel.Tag) string {
+	return util.HashtagFinderRegex.ReplaceAllStringFunc(in, func(match string) string {
+		for _, tag := range tags {
+			if strings.TrimSpace(match) == fmt.Sprintf("#%s", tag.Name) {
+				tagContent := fmt.Sprintf(`<a href="%s" class="mention hashtag" rel="tag">#<span>%s</span></a>`, tag.URL, tag.Name)
+				if strings.HasPrefix(match, " ") {
+					tagContent = " " + tagContent
+				}
+				return tagContent
+			}
+		}
+		return in
+	})
+}
+
+func (f *formatter) ReplaceMentions(in string, mentions []*gtsmodel.Mention) string {
+	for _, menchie := range mentions {
+		targetAccount := &gtsmodel.Account{}
+		if err := f.db.GetByID(menchie.TargetAccountID, targetAccount); err == nil {
+			mentionContent := fmt.Sprintf(`<span class="h-card"><a href="%s" class="u-url mention">@<span>%s</span></a></span>`, targetAccount.URL, targetAccount.Username)
+			in = strings.ReplaceAll(in, menchie.NameString, mentionContent)
+		}
+	}
+	return in
+}

internal/text/formatter.go

@@ -31,6 +31,13 @@ type Formatter interface {
 	FromMarkdown(md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
 	// FromPlain parses an HTML text from a plaintext.
 	FromPlain(plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
+
+	// ReplaceTags takes a piece of text and a slice of tags, and returns the same text with the tags nicely formatted as hrefs.
+	ReplaceTags(in string, tags []*gtsmodel.Tag) string
+	// ReplaceMentions takes a piece of text and a slice of mentions, and returns the same text with the mentions nicely formatted as hrefs.
+	ReplaceMentions(in string, mentions []*gtsmodel.Mention) string
+	// ReplaceLinks takes a piece of text, finds all recognizable links in that text, and replaces them with hrefs.
+	ReplaceLinks(in string) string
 }
 
 type formatter struct {

internal/text/formatter_test.go

@@ -0,0 +1,51 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors [email protected]
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package text_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/text"
+)
+
+// nolint
+type TextStandardTestSuite struct {
+	// standard suite interfaces
+	suite.Suite
+	config *config.Config
+	db     db.DB
+	log    *logrus.Logger
+
+	// standard suite models
+	testTokens       map[string]*oauth.Token
+	testClients      map[string]*oauth.Client
+	testApplications map[string]*gtsmodel.Application
+	testUsers        map[string]*gtsmodel.User
+	testAccounts     map[string]*gtsmodel.Account
+	testAttachments  map[string]*gtsmodel.MediaAttachment
+	testStatuses     map[string]*gtsmodel.Status
+	testTags         map[string]*gtsmodel.Tag
+
+	// module being tested
+	formatter text.Formatter
+}

internal/text/link.go

@@ -82,7 +82,7 @@ func contains(urls []*url.URL, url *url.URL) bool {
 // Note: because Go doesn't allow negative lookbehinds in regex, it's possible that an already-formatted
 // href will end up double-formatted, if the text you pass here contains one or more hrefs already.
 // To avoid this, you should sanitize any HTML out of text before you pass it into this function.
-func ReplaceLinks(in string) string {
+func (f *formatter) ReplaceLinks(in string) string {
 	rxStrict, err := xurls.StrictMatchingScheme(schemes)
 	if err != nil {
 		panic(err)