From fc2b641a0de7c46a6076e23817de597fabcb50fa Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sat, 30 Mar 2024 15:21:20 -0700 Subject: [PATCH 01/11] Remove dead code --- internal/processing/common/status.go | 81 ---------------------------- 1 file changed, 81 deletions(-) diff --git a/internal/processing/common/status.go b/internal/processing/common/status.go index 308f5173f6..2bd269bdc8 100644 --- a/internal/processing/common/status.go +++ b/internal/processing/common/status.go @@ -192,87 +192,6 @@ func (p *Processor) GetAPIStatus( return apiStatus, nil } -// GetVisibleAPIStatuses converts an array of gtsmodel.Status (inputted by next function) into -// API model statuses, checking first for visibility. Please note that all errors will be -// logged at ERROR level, but will not be returned. Callers are likely to run into show-stopping -// errors in the lead-up to this function, whereas calling this should not be a show-stopper. -func (p *Processor) GetVisibleAPIStatuses( - ctx context.Context, - requester *gtsmodel.Account, - next func(int) *gtsmodel.Status, - length int, -) []*apimodel.Status { - return p.getVisibleAPIStatuses(ctx, 3, requester, next, length) -} - -// GetVisibleAPIStatusesPaged is functionally equivalent to GetVisibleAPIStatuses(), -// except the statuses are returned as a converted slice of statuses as interface{}. -func (p *Processor) GetVisibleAPIStatusesPaged( - ctx context.Context, - requester *gtsmodel.Account, - next func(int) *gtsmodel.Status, - length int, -) []interface{} { - statuses := p.getVisibleAPIStatuses(ctx, 3, requester, next, length) - if len(statuses) == 0 { - return nil - } - items := make([]interface{}, len(statuses)) - for i, status := range statuses { - items[i] = status - } - return items -} - -func (p *Processor) getVisibleAPIStatuses( - ctx context.Context, - calldepth int, // used to skip wrapping func above these's names - requester *gtsmodel.Account, - next func(int) *gtsmodel.Status, - length int, -) []*apimodel.Status { - // Start new log entry with - // the above calling func's name. - l := log. - WithContext(ctx). - WithField("caller", log.Caller(calldepth+1)) - - // Preallocate slice according to expected length. - statuses := make([]*apimodel.Status, 0, length) - - for i := 0; i < length; i++ { - // Get next status. - status := next(i) - if status == nil { - continue - } - - // Check whether this status is visible to requesting account. - visible, err := p.filter.StatusVisible(ctx, requester, status) - if err != nil { - l.Errorf("error checking status visibility: %v", err) - continue - } - - if !visible { - // Not visible to requester. - continue - } - - // Convert the status to an API model representation. - apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requester) - if err != nil { - l.Errorf("error converting status: %v", err) - continue - } - - // Append API model to return slice. - statuses = append(statuses, apiStatus) - } - - return statuses -} - // InvalidateTimelinedStatus is a shortcut function for invalidating the cached // representation one status in the home timeline and all list timelines of the // given accountID. It should only be called in cases where a status update From 125b4331ede37aa1319e28a33446bcdbd98164e4 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 14:01:48 -0700 Subject: [PATCH 02/11] Filter statuses when converting to frontend representation --- internal/api/model/filterresult.go | 34 +++ internal/api/model/filterv2.go | 106 ++++++++ internal/api/model/status.go | 2 + internal/filter/custom/user.go | 45 ++++ internal/processing/account/bookmarks.go | 3 +- internal/processing/account/statuses.go | 9 +- internal/processing/common/status.go | 2 +- internal/processing/search/util.go | 2 +- internal/processing/status/get.go | 11 +- .../processing/stream/statusupdate_test.go | 3 +- internal/processing/timeline/faved.go | 3 +- internal/processing/timeline/home.go | 9 +- internal/processing/timeline/list.go | 9 +- internal/processing/timeline/notification.go | 16 +- internal/processing/timeline/public.go | 16 +- internal/processing/timeline/tag.go | 12 +- .../processing/workers/fromclientapi_test.go | 5 +- internal/processing/workers/surfacenotify.go | 2 +- .../processing/workers/surfacetimeline.go | 29 +- internal/timeline/prepare.go | 7 + internal/typeutils/converter_test.go | 26 +- internal/typeutils/frontendtointernal.go | 10 + internal/typeutils/internaltofrontend.go | 255 ++++++++++++++++-- internal/typeutils/internaltofrontend_test.go | 180 ++++++++++++- 24 files changed, 747 insertions(+), 49 deletions(-) create mode 100644 internal/api/model/filterresult.go create mode 100644 internal/api/model/filterv2.go create mode 100644 internal/filter/custom/user.go diff --git a/internal/api/model/filterresult.go b/internal/api/model/filterresult.go new file mode 100644 index 0000000000..942c2124a0 --- /dev/null +++ b/internal/api/model/filterresult.go @@ -0,0 +1,34 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +// FilterResult is returned along with a filtered status to explain why it was filtered. +// +// swagger:model filterResult +// +// --- +// tags: +// - filters +type FilterResult struct { + // The filter that was matched. + Filter FilterV2 `json:"filter"` + // The keywords within the filter that were matched. + KeywordMatches []string `json:"keyword_matches"` + // The status IDs within the filter that were matched. + StatusMatches []string `json:"status_matches"` +} diff --git a/internal/api/model/filterv2.go b/internal/api/model/filterv2.go new file mode 100644 index 0000000000..cf7a129c36 --- /dev/null +++ b/internal/api/model/filterv2.go @@ -0,0 +1,106 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +// FilterV2 represents a user-defined filter for determining which statuses should not be shown to the user. +// v2 filters have names and can include multiple phrases and status IDs to filter. +// +// swagger:model filterV2 +// +// --- +// tags: +// - filters +type FilterV2 struct { + // The ID of the filter in the database. + ID string `json:"id"` + // The name of the filter. + // + // Example: Linux Words + Title string `json:"title"` + // The contexts in which the filter should be applied. + // + // Minimum items: 1 + // Unique: true + // Enum: + // - home + // - notifications + // - public + // - thread + // - account + // Example: ["home", "public"] + Context []FilterContext `json:"context"` + // When the filter should no longer be applied. Null if the filter does not expire. + // + // Example: 2024-02-01T02:57:49Z + ExpiresAt *string `json:"expires_at"` + // The action to be taken when a status matches this filter. + // Enum: + // - warn + // - hide + FilterAction FilterAction `json:"filter_action"` + // The keywords grouped under this filter. + Keywords []FilterKeyword `json:"keywords"` + // The statuses grouped under this filter. + Statuses []FilterStatus `json:"statuses"` +} + +// FilterAction is the action to apply to statuses matching a filter. +type FilterAction string + +const ( + // FilterActionWarn filters will include this status in API results with a warning. + FilterActionWarn FilterAction = "warn" + // FilterActionHide filters will remove this status from API results. + FilterActionHide FilterAction = "hide" + + FilterActionNumValues = 2 +) + +// FilterKeyword represents text to filter within a v2 filter. +// +// swagger:model filterKeyword +// +// --- +// tags: +// - filters +type FilterKeyword struct { + // The ID of the filter keyword entry in the database. + ID string `json:"id"` + // The text to be filtered. + // + // Example: fnord + Keyword string `json:"keyword"` + // Should the filter consider word boundaries? + // + // Example: true + WholeWord bool `json:"whole_word"` +} + +// FilterStatus represents a single status to filter within a v2 filter. +// +// swagger:model filterStatus +// +// --- +// tags: +// - filters +type FilterStatus struct { + // The ID of the filter status entry in the database. + ID string `json:"id"` + // The status ID to be filtered. + StatusID string `json:"phrase"` +} diff --git a/internal/api/model/status.go b/internal/api/model/status.go index fed2cdf379..d27f38b5d2 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -100,6 +100,8 @@ type Status struct { // so the user may redraft from the source text without the client having to reverse-engineer // the original text from the HTML content. Text string `json:"text,omitempty"` + // A filter that matched this status and why it matched, if there is such a filter. + Filtered *FilterResult `json:"filtered,omitempty"` // Additional fields not exposed via JSON // (used only internally for templating etc). diff --git a/internal/filter/custom/user.go b/internal/filter/custom/user.go new file mode 100644 index 0000000000..3b0eaa299f --- /dev/null +++ b/internal/filter/custom/user.go @@ -0,0 +1,45 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package custom represents custom filters managed by the user through the API. +package custom + +import ( + "errors" +) + +// HideStatus indicates that a status has been filtered and should not be returned at all. +var HideStatus = errors.New("hide status") + +// FilterContext determines the filters that apply to a given status or list of statuses. +type FilterContext string + +const ( + // FilterContextNone means no filters should be applied. + // There are no filters with this context; it's for internal use only. + FilterContextNone FilterContext = "" + // FilterContextHome means this status is being filtered as part of a home or list timeline. + FilterContextHome FilterContext = "home" + // FilterContextNotifications means this status is being filtered as part of the notifications timeline. + FilterContextNotifications FilterContext = "notifications" + // FilterContextPublic means this status is being filtered as part of a public or tag timeline. + FilterContextPublic FilterContext = "public" + // FilterContextThread means this status is being filtered as part of a thread's context. + FilterContextThread FilterContext = "thread" + // FilterContextAccount means this status is being filtered as part of an account's statuses. + FilterContextAccount FilterContext = "account" +) diff --git a/internal/processing/account/bookmarks.go b/internal/processing/account/bookmarks.go index 9cbc3db26d..d934c9b4e6 100644 --- a/internal/processing/account/bookmarks.go +++ b/internal/processing/account/bookmarks.go @@ -23,6 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -74,7 +75,7 @@ func (p *Processor) BookmarksGet(ctx context.Context, requestingAccount *gtsmode } // Convert the status. - item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount) + item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextNone, nil) if err != nil { log.Errorf(ctx, "error converting bookmarked status to api: %s", err) continue diff --git a/internal/processing/account/statuses.go b/internal/processing/account/statuses.go index 0985bb4ef6..6cf519044a 100644 --- a/internal/processing/account/statuses.go +++ b/internal/processing/account/statuses.go @@ -24,6 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -96,9 +97,15 @@ func (p *Processor) StatusesGet( return nil, gtserror.NewErrorInternalError(err) } + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + for _, s := range filtered { // Convert filtered statuses to API statuses. - item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount) + item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, custom.FilterContextAccount, filters) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/common/status.go b/internal/processing/common/status.go index 2bd269bdc8..7aa8363069 100644 --- a/internal/processing/common/status.go +++ b/internal/processing/common/status.go @@ -184,7 +184,7 @@ func (p *Processor) GetAPIStatus( apiStatus *apimodel.Status, errWithCode gtserror.WithCode, ) { - apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester, "", nil) if err != nil { err = gtserror.Newf("error converting status: %w", err) return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/search/util.go b/internal/processing/search/util.go index de91e5d513..60714f95e1 100644 --- a/internal/processing/search/util.go +++ b/internal/processing/search/util.go @@ -113,7 +113,7 @@ func (p *Processor) packageStatuses( continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, "", nil) if err != nil { log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", status.ID, err) continue diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 7256d2f82f..123b58a3f8 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -23,6 +23,7 @@ import ( "strings" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -210,7 +211,15 @@ func TopoSort(apiStatuses []*apimodel.Status, targetAccountID string) { // ContextGet returns the context (previous and following posts) from the given status ID. func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode) { - return p.contextGet(ctx, requestingAccount, targetStatusID, p.converter.StatusToAPIStatus) + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + convert := func(ctx context.Context, status *gtsmodel.Status, requestingAccount *gtsmodel.Account) (*apimodel.Status, error) { + return p.converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextThread, filters) + } + return p.contextGet(ctx, requestingAccount, targetStatusID, convert) } // WebContextGet is like ContextGet, but is explicitly diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index 8814c966ff..e0417b0c92 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/stream" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -39,7 +40,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.NoError(errWithCode) editedStatus := suite.testStatuses["remote_account_1_status_1"] - apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account) + apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account, custom.FilterContextNotifications, nil) suite.NoError(err) suite.streamProcessor.StatusUpdate(context.Background(), account, apiStatus, stream.TimelineHome) diff --git a/internal/processing/timeline/faved.go b/internal/processing/timeline/faved.go index 205b15069a..cf5c0c6143 100644 --- a/internal/processing/timeline/faved.go +++ b/internal/processing/timeline/faved.go @@ -24,6 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/oauth" @@ -54,7 +55,7 @@ func (p *Processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, ma continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, custom.FilterContextNone, nil) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/timeline/home.go b/internal/processing/timeline/home.go index d12dd98c48..3b86f3555e 100644 --- a/internal/processing/timeline/home.go +++ b/internal/processing/timeline/home.go @@ -23,6 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -98,7 +99,13 @@ func HomeTimelineStatusPrepare(state *state.State, converter *typeutils.Converte return nil, err } - return converter.StatusToAPIStatus(ctx, status, requestingAccount) + filters, err := state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, err + } + + return converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextHome, filters) } } diff --git a/internal/processing/timeline/list.go b/internal/processing/timeline/list.go index 7356d1978b..4703bf941f 100644 --- a/internal/processing/timeline/list.go +++ b/internal/processing/timeline/list.go @@ -23,6 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -110,7 +111,13 @@ func ListTimelineStatusPrepare(state *state.State, converter *typeutils.Converte return nil, err } - return converter.StatusToAPIStatus(ctx, status, requestingAccount) + filters, err := state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, err + } + + return converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextHome, filters) } } diff --git a/internal/processing/timeline/notification.go b/internal/processing/timeline/notification.go index 09febdb469..f31e40b32b 100644 --- a/internal/processing/timeline/notification.go +++ b/internal/processing/timeline/notification.go @@ -43,6 +43,12 @@ func (p *Processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, ma return util.EmptyPageableResponse(), nil } + filters, err := p.state.DB.GetFiltersForAccountID(ctx, authed.Account.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", authed.Account.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + var ( items = make([]interface{}, 0, count) nextMaxIDValue string @@ -87,7 +93,7 @@ func (p *Processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, ma } } - item, err := p.converter.NotificationToAPINotification(ctx, n) + item, err := p.converter.NotificationToAPINotification(ctx, n, filters) if err != nil { log.Debugf(ctx, "skipping notification %s because it couldn't be converted to its api representation: %s", n.ID, err) continue @@ -121,7 +127,13 @@ func (p *Processor) NotificationGet(ctx context.Context, account *gtsmodel.Accou return nil, gtserror.NewErrorNotFound(err) } - apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif) + filters, err := p.state.DB.GetFiltersForAccountID(ctx, account.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", account.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + + apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, filters) if err != nil { if errors.Is(err, db.ErrNoEntries) { return nil, gtserror.NewErrorNotFound(err) diff --git a/internal/processing/timeline/public.go b/internal/processing/timeline/public.go index 87de04f4a0..12c4c7958a 100644 --- a/internal/processing/timeline/public.go +++ b/internal/processing/timeline/public.go @@ -24,6 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -46,6 +47,16 @@ func (p *Processor) PublicTimelineGet( items = make([]any, 0, limit) ) + var filters []*gtsmodel.Filter + if requester != nil { + var err error + filters, err = p.state.DB.GetFiltersForAccountID(ctx, requester.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requester.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + } + // Try a few times to select appropriate public // statuses from the db, paging up or down to // reattempt if nothing suitable is found. @@ -87,7 +98,10 @@ outer: continue inner } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester, custom.FilterContextPublic, filters) + if errors.Is(err, custom.HideStatus) { + continue + } if err != nil { log.Errorf(ctx, "error converting to api status: %v", err) continue inner diff --git a/internal/processing/timeline/tag.go b/internal/processing/timeline/tag.go index 45632ce069..0d583a1e92 100644 --- a/internal/processing/timeline/tag.go +++ b/internal/processing/timeline/tag.go @@ -24,6 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -111,6 +112,12 @@ func (p *Processor) packageTagResponse( prevMinIDValue = statuses[0].ID ) + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAcct.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAcct.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + for _, s := range statuses { timelineable, err := p.filter.StatusTagTimelineable(ctx, requestingAcct, s) if err != nil { @@ -122,7 +129,10 @@ func (p *Processor) packageTagResponse( continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct, custom.FilterContextPublic, filters) + if errors.Is(err, custom.HideStatus) { + continue + } if err != nil { log.Errorf(ctx, "error converting to api status: %v", err) continue diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index 3d3630b11a..742e118eca 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -28,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/messages" @@ -150,6 +151,8 @@ func (suite *FromClientAPITestSuite) statusJSON( ctx, status, requestingAccount, + custom.FilterContextNone, + nil, ) if err != nil { suite.FailNow(err.Error()) @@ -244,7 +247,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() { suite.FailNow("timed out waiting for new status notification") } - apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notif) + apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notif, nil) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/processing/workers/surfacenotify.go b/internal/processing/workers/surfacenotify.go index a8c36248c7..dff687235d 100644 --- a/internal/processing/workers/surfacenotify.go +++ b/internal/processing/workers/surfacenotify.go @@ -390,7 +390,7 @@ func (s *surface) notify( } // Stream notification to the user. - apiNotif, err := s.converter.NotificationToAPINotification(ctx, notif) + apiNotif, err := s.converter.NotificationToAPINotification(ctx, notif, nil) if err != nil { return gtserror.Newf("error converting notification to api representation: %w", err) } diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index 14634f8463..a563bed12f 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -111,6 +112,11 @@ func (s *surface) timelineAndNotifyStatusForFollowers( continue } + filters, err := s.state.DB.GetFiltersForAccountID(ctx, follow.AccountID) + if err != nil { + return gtserror.Newf("couldn't retrieve filters for account %s: %w", follow.AccountID, err) + } + // Add status to any relevant lists // for this follow, if applicable. s.listTimelineStatusForFollow( @@ -118,6 +124,7 @@ func (s *surface) timelineAndNotifyStatusForFollowers( status, follow, &errs, + filters, ) // Add status to home timeline for owner @@ -129,6 +136,7 @@ func (s *surface) timelineAndNotifyStatusForFollowers( follow.Account, status, stream.TimelineHome, + filters, ) if err != nil { errs.Appendf("error home timelining status: %w", err) @@ -180,6 +188,7 @@ func (s *surface) listTimelineStatusForFollow( status *gtsmodel.Status, follow *gtsmodel.Follow, errs *gtserror.MultiError, + filters []*gtsmodel.Filter, ) { // To put this status in appropriate list timelines, // we need to get each listEntry that pertains to @@ -222,6 +231,7 @@ func (s *surface) listTimelineStatusForFollow( follow.Account, status, stream.TimelineList+":"+listEntry.ListID, // key streamType to this specific list + filters, ); err != nil { errs.Appendf("error adding status to timeline for list %s: %w", listEntry.ListID, err) // implicit continue @@ -332,6 +342,7 @@ func (s *surface) timelineStatus( account *gtsmodel.Account, status *gtsmodel.Status, streamType string, + filters []*gtsmodel.Filter, ) (bool, error) { // Ingest status into given timeline using provided function. if inserted, err := ingest(ctx, timelineID, status); err != nil { @@ -343,7 +354,7 @@ func (s *surface) timelineStatus( } // The status was inserted so stream it to the user. - apiStatus, err := s.converter.StatusToAPIStatus(ctx, status, account) + apiStatus, err := s.converter.StatusToAPIStatus(ctx, status, account, custom.FilterContextHome, filters) if err != nil { err = gtserror.Newf("error converting status %s to frontend representation: %w", status.ID, err) return true, err @@ -457,6 +468,11 @@ func (s *surface) timelineStatusUpdateForFollowers( continue } + filters, err := s.state.DB.GetFiltersForAccountID(ctx, follow.AccountID) + if err != nil { + return gtserror.Newf("couldn't retrieve filters for account %s: %w", follow.AccountID, err) + } + // Add status to any relevant lists // for this follow, if applicable. s.listTimelineStatusUpdateForFollow( @@ -464,6 +480,7 @@ func (s *surface) timelineStatusUpdateForFollowers( status, follow, &errs, + filters, ) // Add status to home timeline for owner @@ -473,6 +490,7 @@ func (s *surface) timelineStatusUpdateForFollowers( follow.Account, status, stream.TimelineHome, + filters, ) if err != nil { errs.Appendf("error home timelining status: %w", err) @@ -490,6 +508,7 @@ func (s *surface) listTimelineStatusUpdateForFollow( status *gtsmodel.Status, follow *gtsmodel.Follow, errs *gtserror.MultiError, + filters []*gtsmodel.Filter, ) { // To put this status in appropriate list timelines, // we need to get each listEntry that pertains to @@ -530,6 +549,7 @@ func (s *surface) listTimelineStatusUpdateForFollow( follow.Account, status, stream.TimelineList+":"+listEntry.ListID, // key streamType to this specific list + filters, ); err != nil { errs.Appendf("error adding status to timeline for list %s: %w", listEntry.ListID, err) // implicit continue @@ -544,8 +564,13 @@ func (s *surface) timelineStreamStatusUpdate( account *gtsmodel.Account, status *gtsmodel.Status, streamType string, + filters []*gtsmodel.Filter, ) error { - apiStatus, err := s.converter.StatusToAPIStatus(ctx, status, account) + apiStatus, err := s.converter.StatusToAPIStatus(ctx, status, account, custom.FilterContextHome, filters) + if errors.Is(err, custom.HideStatus) { + // Don't put this status in the stream. + return nil + } if err != nil { err = gtserror.Newf("error converting status %s to frontend representation: %w", status.ID, err) return err diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go index 07bde79fab..769c94144c 100644 --- a/internal/timeline/prepare.go +++ b/internal/timeline/prepare.go @@ -24,6 +24,7 @@ import ( "codeberg.org/gruf/go-kv" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" ) @@ -121,6 +122,12 @@ func (t *timeline) prepareXBetweenIDs(ctx context.Context, amount int, behindID for e, entry := range toPrepare { prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) if err != nil { + if errors.Is(err, custom.HideStatus) { + // This item has been filtered out by the requesting user's filters. + // Remove it and skip past it. + t.items.data.Remove(e) + continue + } if errors.Is(err, db.ErrNoEntries) { // ErrNoEntries means something has been deleted, // so we'll likely not be able to ever prepare this. diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index 716a39c295..fc873a94bd 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -473,16 +473,19 @@ const ( type TypeUtilsTestSuite struct { suite.Suite - db db.DB - state state.State - testAccounts map[string]*gtsmodel.Account - testStatuses map[string]*gtsmodel.Status - testAttachments map[string]*gtsmodel.MediaAttachment - testPeople map[string]vocab.ActivityStreamsPerson - testEmojis map[string]*gtsmodel.Emoji - testReports map[string]*gtsmodel.Report - testMentions map[string]*gtsmodel.Mention - testPollVotes map[string]*gtsmodel.PollVote + db db.DB + state state.State + testAccounts map[string]*gtsmodel.Account + testStatuses map[string]*gtsmodel.Status + testAttachments map[string]*gtsmodel.MediaAttachment + testPeople map[string]vocab.ActivityStreamsPerson + testEmojis map[string]*gtsmodel.Emoji + testReports map[string]*gtsmodel.Report + testMentions map[string]*gtsmodel.Mention + testPollVotes map[string]*gtsmodel.PollVote + testFilters map[string]*gtsmodel.Filter + testFilterKeywords map[string]*gtsmodel.FilterKeyword + testFilterStatues map[string]*gtsmodel.FilterStatus typeconverter *typeutils.Converter } @@ -506,6 +509,9 @@ func (suite *TypeUtilsTestSuite) SetupTest() { suite.testReports = testrig.NewTestReports() suite.testMentions = testrig.NewTestMentions() suite.testPollVotes = testrig.NewTestPollVotes() + suite.testFilters = testrig.NewTestFilters() + suite.testFilterKeywords = testrig.NewTestFilterKeywords() + suite.testFilterStatues = testrig.NewTestFilterStatuses() suite.typeconverter = typeutils.NewConverter(&suite.state) testrig.StandardDBSetup(suite.db, nil) diff --git a/internal/typeutils/frontendtointernal.go b/internal/typeutils/frontendtointernal.go index 3bb0933f32..dc8174a203 100644 --- a/internal/typeutils/frontendtointernal.go +++ b/internal/typeutils/frontendtointernal.go @@ -47,3 +47,13 @@ func APIMarkerNameToMarkerName(m apimodel.MarkerName) gtsmodel.MarkerName { } return "" } + +func APIFilterActionToFilterAction(m apimodel.FilterAction) gtsmodel.FilterAction { + switch m { + case apimodel.FilterActionWarn: + return gtsmodel.FilterActionWarn + case apimodel.FilterActionHide: + return gtsmodel.FilterActionHide + } + return "" +} diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index bf44c7254b..87c29a403b 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -22,12 +22,15 @@ import ( "errors" "fmt" "math" + "regexp" "strconv" "strings" + "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/language" @@ -676,12 +679,19 @@ func (c *Converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistor // (frontend) representation for serialization on the API. // // Requesting account can be nil. +// +// Filter context can be the empty string if these statuses are not being filtered. +// +// If there is a matching "hide" filter, the returned status will be nil with a HideStatus error; +// callers need to handle that case by excluding it from results. func (c *Converter) StatusToAPIStatus( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, + filterContext custom.FilterContext, + filters []*gtsmodel.Filter, ) (*apimodel.Status, error) { - apiStatus, err := c.statusToFrontend(ctx, s, requestingAccount) + apiStatus, err := c.statusToFrontend(ctx, s, requestingAccount, filterContext, filters) if err != nil { return nil, err } @@ -696,6 +706,154 @@ func (c *Converter) StatusToAPIStatus( return apiStatus, nil } +// statusToAPIFilterResult applies filters to a status and returns an API filter result object. +// The result may be nil if no filters matched. +// If the status should not be returned at all, it returns the HideStatus error. +func (c *Converter) statusToAPIFilterResult( + ctx context.Context, + s *gtsmodel.Status, + requestingAccount *gtsmodel.Account, + filterContext custom.FilterContext, + filters []*gtsmodel.Filter, +) (*apimodel.FilterResult, error) { + if filterContext == "" || len(filters) == 0 || s.AccountID == requestingAccount.ID { + return nil, nil + } + + var filterResult *apimodel.FilterResult + + now := time.Now() + for _, filter := range filters { + if !filterAppliesInContext(filter, filterContext) { + // Filter doesn't apply to this context. + continue + } + if !filter.ExpiresAt.IsZero() && filter.ExpiresAt.Before(now) { + // Filter is expired. + continue + } + if filterResult != nil && filter.Action == gtsmodel.FilterActionWarn { + // Filter is a warn filter, but we've already matched at least one of those. + continue + } + + // List all matching keywords. + // TODO: (Vyr) this might be sped up slightly by concatenating regexps in a cache somewhere, + // although we still have to keep track of which actual keywords matched. + keywordMatches := make([]string, 0, len(filter.Keywords)) + fields := filterableTextFields(s) + for _, filterKeyword := range filter.Keywords { + wholeWord := util.PtrValueOr(filterKeyword.WholeWord, false) + var isMatch bool + for _, field := range fields { + if wholeWord { + regexpIsMatch, err := regexp.MatchString(`\b`+regexp.QuoteMeta(filterKeyword.Keyword)+`\b`, field) + if err != nil { + return nil, err + } + if regexpIsMatch { + isMatch = true + } + } else { + if strings.Contains(field, filterKeyword.Keyword) { + isMatch = true + } + } + } + if isMatch { + keywordMatches = append(keywordMatches, filterKeyword.Keyword) + } + } + + // A status has only one ID. Not clear why this is a list in the Mastodon API. + // TODO: (Vyr) Likewise, there could be a set of status IDs in a cache. + statusMatches := make([]string, 0, 1) + for _, filterStatus := range filter.Statuses { + if s.ID == filterStatus.StatusID { + statusMatches = append(statusMatches, filterStatus.StatusID) + break + } + } + + if len(keywordMatches) > 0 || len(statusMatches) > 0 { + switch filter.Action { + case gtsmodel.FilterActionWarn: + if filterResult == nil { + // If this is the first filter to match, record what matched. + apiFilter, err := c.FilterToAPIFilterV2(ctx, filter) + if err != nil { + return nil, err + } + filterResult = &apimodel.FilterResult{ + Filter: *apiFilter, + KeywordMatches: keywordMatches, + StatusMatches: statusMatches, + } + } + // We'll keep going after this in case there's a filter with a hide action that also matches. + + case gtsmodel.FilterActionHide: + // Don't show this status. Immediate return. + return nil, custom.HideStatus + } + } + } + + return filterResult, nil +} + +// filterableTextFields returns all text from a status that we might want to filter on: +// - content +// - content warning +// - media descriptions +// - poll options +func filterableTextFields(s *gtsmodel.Status) []string { + fieldCount := 2 + len(s.Attachments) + if s.Poll != nil { + fieldCount += len(s.Poll.Options) + } + fields := make([]string, 0, fieldCount) + + if s.Content != "" { + // TODO: (Vyr) convert this HTML field to plain text before returning + fields = append(fields, s.Content) + } + if s.ContentWarning != "" { + fields = append(fields, s.ContentWarning) + } + for _, attachment := range s.Attachments { + if attachment.Description != "" { + fields = append(fields, attachment.Description) + } + } + if s.Poll != nil { + for _, option := range s.Poll.Options { + if option != "" { + fields = append(fields, option) + } + } + } + + return fields +} + +// filterAppliesInContext returns whether a given filter applies in a given context. +func filterAppliesInContext(filter *gtsmodel.Filter, filterContext custom.FilterContext) bool { + switch filterContext { + case custom.FilterContextHome: + return util.PtrValueOr(filter.ContextHome, false) + case custom.FilterContextNotifications: + return util.PtrValueOr(filter.ContextNotifications, false) + case custom.FilterContextPublic: + return util.PtrValueOr(filter.ContextPublic, false) + case custom.FilterContextThread: + return util.PtrValueOr(filter.ContextThread, false) + case custom.FilterContextAccount: + return util.PtrValueOr(filter.ContextAccount, false) + } + return false +} + // StatusToWebStatus converts a gts model status into an // api representation suitable for serving into a web template. // @@ -705,7 +863,8 @@ func (c *Converter) StatusToWebStatus( s *gtsmodel.Status, requestingAccount *gtsmodel.Account, ) (*apimodel.Status, error) { - webStatus, err := c.statusToFrontend(ctx, s, requestingAccount) + // TODO: (Vyr) it's not clear to me why we'd have an account when requesting a web status + webStatus, err := c.statusToFrontend(ctx, s, requestingAccount, "", nil) if err != nil { return nil, err } @@ -792,6 +951,8 @@ func (c *Converter) statusToFrontend( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, + filterContext custom.FilterContext, + filters []*gtsmodel.Filter, ) (*apimodel.Status, error) { // Try to populate status struct pointer fields. // We can continue in many cases of partial failure, @@ -903,7 +1064,11 @@ func (c *Converter) statusToFrontend( } if s.BoostOf != nil { - reblog, err := c.StatusToAPIStatus(ctx, s.BoostOf, requestingAccount) + reblog, err := c.StatusToAPIStatus(ctx, s.BoostOf, requestingAccount, filterContext, filters) + if errors.Is(err, custom.HideStatus) { + // If we'd hide the original status, hide the boost. + return nil, err + } if err != nil { return nil, gtserror.Newf("error converting boosted status: %w", err) } @@ -939,6 +1104,13 @@ func (c *Converter) statusToFrontend( s.URL = s.URI } + // Apply filters. + filterResult, err := c.statusToAPIFilterResult(ctx, s, requestingAccount, filterContext, filters) + if err != nil { + return nil, fmt.Errorf("error applying filters: %w", err) + } + apiStatus.Filtered = filterResult + return apiStatus, nil } @@ -1214,7 +1386,7 @@ func (c *Converter) RelationshipToAPIRelationship(ctx context.Context, r *gtsmod } // NotificationToAPINotification converts a gts notification into a api notification -func (c *Converter) NotificationToAPINotification(ctx context.Context, n *gtsmodel.Notification) (*apimodel.Notification, error) { +func (c *Converter) NotificationToAPINotification(ctx context.Context, n *gtsmodel.Notification, filters []*gtsmodel.Filter) (*apimodel.Notification, error) { if n.TargetAccount == nil { tAccount, err := c.state.DB.GetAccountByID(ctx, n.TargetAccountID) if err != nil { @@ -1255,7 +1427,7 @@ func (c *Converter) NotificationToAPINotification(ctx context.Context, n *gtsmod } var err error - apiStatus, err = c.StatusToAPIStatus(ctx, n.Status, n.TargetAccount) + apiStatus, err = c.StatusToAPIStatus(ctx, n.Status, n.TargetAccount, custom.FilterContextNotifications, filters) if err != nil { return nil, fmt.Errorf("NotificationToapi: error converting status to api: %s", err) } @@ -1408,7 +1580,7 @@ func (c *Converter) ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Repo } } for _, s := range r.Statuses { - status, err := c.StatusToAPIStatus(ctx, s, requestingAccount) + status, err := c.StatusToAPIStatus(ctx, s, requestingAccount, "", nil) if err != nil { return nil, fmt.Errorf("ReportToAdminAPIReport: error converting status with id %s to api status: %w", s.ID, err) } @@ -1649,6 +1821,55 @@ func (c *Converter) FilterKeywordToAPIFilterV1(ctx context.Context, filterKeywor } filter := filterKeyword.Filter + return &apimodel.FilterV1{ + // v1 filters have a single keyword each, so we use the filter keyword ID as the v1 filter ID. + ID: filterKeyword.ID, + Phrase: filterKeyword.Keyword, + Context: filterToAPIFilterContexts(filter), + WholeWord: util.PtrValueOr(filterKeyword.WholeWord, false), + ExpiresAt: filterExpiresAtToAPIFilterExpiresAt(filter.ExpiresAt), + Irreversible: filter.Action == gtsmodel.FilterActionHide, + }, nil +} + +// FilterToAPIFilterV2 converts one GTS model filter into an API v2 filter. +func (c *Converter) FilterToAPIFilterV2(ctx context.Context, filter *gtsmodel.Filter) (*apimodel.FilterV2, error) { + apiFilterKeywords := make([]apimodel.FilterKeyword, 0, len(filter.Keywords)) + for _, filterKeyword := range filter.Keywords { + apiFilterKeywords = append(apiFilterKeywords, apimodel.FilterKeyword{ + ID: filterKeyword.ID, + Keyword: filterKeyword.Keyword, + WholeWord: util.PtrValueOr(filterKeyword.WholeWord, false), + }) + } + + apiFilterStatuses := make([]apimodel.FilterStatus, 0, len(filter.Keywords)) + for _, filterStatus := range filter.Statuses { + apiFilterStatuses = append(apiFilterStatuses, apimodel.FilterStatus{ + ID: filterStatus.ID, + StatusID: filterStatus.StatusID, + }) + } + + return &apimodel.FilterV2{ + ID: filter.ID, + Title: filter.Title, + Context: filterToAPIFilterContexts(filter), + ExpiresAt: filterExpiresAtToAPIFilterExpiresAt(filter.ExpiresAt), + FilterAction: filterActionToAPIFilterAction(filter.Action), + Keywords: apiFilterKeywords, + Statuses: apiFilterStatuses, + }, nil +} + +func filterExpiresAtToAPIFilterExpiresAt(expiresAt time.Time) *string { + if expiresAt.IsZero() { + return nil + } + return util.Ptr(util.FormatISO8601(expiresAt)) +} + +func filterToAPIFilterContexts(filter *gtsmodel.Filter) []apimodel.FilterContext { apiContexts := make([]apimodel.FilterContext, 0, apimodel.FilterContextNumValues) if util.PtrValueOr(filter.ContextHome, false) { apiContexts = append(apiContexts, apimodel.FilterContextHome) @@ -1665,21 +1886,17 @@ func (c *Converter) FilterKeywordToAPIFilterV1(ctx context.Context, filterKeywor if util.PtrValueOr(filter.ContextAccount, false) { apiContexts = append(apiContexts, apimodel.FilterContextAccount) } + return apiContexts +} - var expiresAt *string - if !filter.ExpiresAt.IsZero() { - expiresAt = util.Ptr(util.FormatISO8601(filter.ExpiresAt)) +func filterActionToAPIFilterAction(m gtsmodel.FilterAction) apimodel.FilterAction { + switch m { + case gtsmodel.FilterActionWarn: + return apimodel.FilterActionWarn + case gtsmodel.FilterActionHide: + return apimodel.FilterActionHide } - - return &apimodel.FilterV1{ - // v1 filters have a single keyword each, so we use the filter keyword ID as the v1 filter ID. - ID: filterKeyword.ID, - Phrase: filterKeyword.Keyword, - Context: apiContexts, - WholeWord: util.PtrValueOr(filterKeyword.WholeWord, false), - ExpiresAt: expiresAt, - Irreversible: filter.Action == gtsmodel.FilterActionHide, - }, nil + return "" } // convertEmojisToAPIEmojis will convert a slice of GTS model emojis to frontend API model emojis, falling back to IDs if no GTS models supplied. diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 519888e211..a630b9c12f 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/custom" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -426,7 +427,7 @@ func (suite *InternalToFrontendTestSuite) TestLocalInstanceAccountToFrontendBloc func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { testStatus := suite.testStatuses["admin_account_status_1"] requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -536,11 +537,184 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { }`, string(b)) } +// Test that a status which is filtered with a warn filter by the requesting user has `filtered` set correctly. +func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { + testStatus := suite.testStatuses["admin_account_status_1"] + testStatus.Content += " fnord" + testStatus.Text += " fnord" + requestingAccount := suite.testAccounts["local_account_1"] + expectedMatchingFilter := suite.testFilters["local_account_1_filter_1"] + expectedMatchingFilterKeyword := suite.testFilterKeywords["local_account_1_filter_1_keyword_1"] + expectedMatchingFilterKeyword.Filter = expectedMatchingFilter + expectedMatchingFilter.Keywords = []*gtsmodel.FilterKeyword{expectedMatchingFilterKeyword} + requestingAccountFilters := []*gtsmodel.Filter{expectedMatchingFilter} + apiStatus, err := suite.typeconverter.StatusToAPIStatus( + context.Background(), + testStatus, + requestingAccount, + custom.FilterContextHome, + requestingAccountFilters, + ) + suite.NoError(err) + + b, err := json.MarshalIndent(apiStatus, "", " ") + suite.NoError(err) + + suite.Equal(`{ + "id": "01F8MH75CBF9JFX4ZAD54N0W0R", + "created_at": "2021-10-20T11:36:45.000Z", + "in_reply_to_id": null, + "in_reply_to_account_id": null, + "sensitive": false, + "spoiler_text": "", + "visibility": "public", + "language": "en", + "uri": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R", + "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R", + "replies_count": 1, + "reblogs_count": 0, + "favourites_count": 1, + "favourited": true, + "reblogged": false, + "muted": false, + "bookmarked": true, + "pinned": false, + "content": "hello world! #welcome ! first post on the instance :rainbow: ! fnord", + "reblog": null, + "application": { + "name": "superseriousbusiness", + "website": "https://superserious.business" + }, + "account": { + "id": "01F8MH17FWEB39HZJ76B6VXSKF", + "username": "admin", + "acct": "admin", + "display_name": "", + "locked": false, + "discoverable": true, + "bot": false, + "created_at": "2022-05-17T13:10:59.000Z", + "note": "", + "url": "http://localhost:8080/@admin", + "avatar": "", + "avatar_static": "", + "header": "http://localhost:8080/assets/default_header.png", + "header_static": "http://localhost:8080/assets/default_header.png", + "followers_count": 1, + "following_count": 1, + "statuses_count": 4, + "last_status_at": "2021-10-20T10:41:37.000Z", + "emojis": [], + "fields": [], + "enable_rss": true, + "role": { + "name": "admin" + } + }, + "media_attachments": [ + { + "id": "01F8MH6NEM8D7527KZAECTCR76", + "type": "image", + "url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", + "text_url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", + "preview_url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg", + "remote_url": null, + "preview_remote_url": null, + "meta": { + "original": { + "width": 1200, + "height": 630, + "size": "1200x630", + "aspect": 1.9047619 + }, + "small": { + "width": 256, + "height": 134, + "size": "256x134", + "aspect": 1.9104477 + }, + "focus": { + "x": 0, + "y": 0 + } + }, + "description": "Black and white image of some 50's style text saying: Welcome On Board", + "blurhash": "LNJRdVM{00Rj%Mayt7j[4nWBofRj" + } + ], + "mentions": [], + "tags": [ + { + "name": "welcome", + "url": "http://localhost:8080/tags/welcome" + } + ], + "emojis": [ + { + "shortcode": "rainbow", + "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png", + "static_url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png", + "visible_in_picker": true, + "category": "reactions" + } + ], + "card": null, + "poll": null, + "text": "hello world! #welcome ! first post on the instance :rainbow: ! fnord", + "filtered": { + "filter": { + "id": "01HN26VM6KZTW1ANNRVSBMA461", + "title": "fnord", + "context": [ + "home", + "public" + ], + "expires_at": null, + "filter_action": "warn", + "keywords": [ + { + "id": "01HN272TAVWAXX72ZX4M8JZ0PS", + "keyword": "fnord", + "whole_word": true + } + ], + "statuses": [] + }, + "keyword_matches": [ + "fnord" + ], + "status_matches": [] + } +}`, string(b)) +} + +// Test that a status which is filtered with a hide filter by the requesting user results in the HideStatus error. +func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() { + testStatus := suite.testStatuses["admin_account_status_1"] + testStatus.Content += " fnord" + testStatus.Text += " fnord" + requestingAccount := suite.testAccounts["local_account_1"] + expectedMatchingFilter := suite.testFilters["local_account_1_filter_1"] + expectedMatchingFilter.Action = gtsmodel.FilterActionHide + expectedMatchingFilterKeyword := suite.testFilterKeywords["local_account_1_filter_1_keyword_1"] + expectedMatchingFilterKeyword.Filter = expectedMatchingFilter + expectedMatchingFilter.Keywords = []*gtsmodel.FilterKeyword{expectedMatchingFilterKeyword} + requestingAccountFilters := []*gtsmodel.Filter{expectedMatchingFilter} + _, err := suite.typeconverter.StatusToAPIStatus( + context.Background(), + testStatus, + requestingAccount, + custom.FilterContextHome, + requestingAccountFilters, + ) + suite.ErrorIs(err, custom.HideStatus) +} + func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments() { testStatus := suite.testStatuses["remote_account_2_status_1"] requestingAccount := suite.testAccounts["admin_account"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -773,7 +947,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Language = "" requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") From be78b7e7012913894826de46cc69489236d98e8c Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 14:55:23 -0700 Subject: [PATCH 03/11] status.filtered is an array --- internal/api/model/status.go | 4 +- internal/typeutils/internaltofrontend.go | 39 +++++++-------- internal/typeutils/internaltofrontend_test.go | 48 ++++++++++--------- 3 files changed, 43 insertions(+), 48 deletions(-) diff --git a/internal/api/model/status.go b/internal/api/model/status.go index d27f38b5d2..86bf6dc870 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -100,8 +100,8 @@ type Status struct { // so the user may redraft from the source text without the client having to reverse-engineer // the original text from the HTML content. Text string `json:"text,omitempty"` - // A filter that matched this status and why it matched, if there is such a filter. - Filtered *FilterResult `json:"filtered,omitempty"` + // A list of filters that matched this status and why they matched, if there are any such filters. + Filtered []FilterResult `json:"filtered,omitempty"` // Additional fields not exposed via JSON // (used only internally for templating etc). diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 87c29a403b..91a8d084da 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -706,21 +706,21 @@ func (c *Converter) StatusToAPIStatus( return apiStatus, nil } -// statusToAPIFilterResult applies filters to a status and returns an API filter result object. +// statusToAPIFilterResults applies filters to a status and returns an API filter result object. // The result may be nil if no filters matched. // If the status should not be returned at all, it returns the HideStatus error. -func (c *Converter) statusToAPIFilterResult( +func (c *Converter) statusToAPIFilterResults( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, filterContext custom.FilterContext, filters []*gtsmodel.Filter, -) (*apimodel.FilterResult, error) { +) ([]apimodel.FilterResult, error) { if filterContext == "" || len(filters) == 0 || s.AccountID == requestingAccount.ID { return nil, nil } - var filterResult *apimodel.FilterResult + filterResults := make([]apimodel.FilterResult, 0, len(filters)) now := time.Now() for _, filter := range filters { @@ -732,10 +732,6 @@ func (c *Converter) statusToAPIFilterResult( // Filter is expired. continue } - if filterResult != nil && filter.Action == gtsmodel.FilterActionWarn { - // Filter is a warn filter, but we've already matched at least one of those. - continue - } // List all matching keywords. // TODO: (Vyr) this might be sped up slightly by concatenating regexps in a cache somewhere, @@ -778,19 +774,16 @@ func (c *Converter) statusToAPIFilterResult( if len(keywordMatches) > 0 || len(statusMatches) > 0 { switch filter.Action { case gtsmodel.FilterActionWarn: - if filterResult == nil { - // If this is the first filter to match, record what matched. - apiFilter, err := c.FilterToAPIFilterV2(ctx, filter) - if err != nil { - return nil, err - } - filterResult = &apimodel.FilterResult{ - Filter: *apiFilter, - KeywordMatches: keywordMatches, - StatusMatches: statusMatches, - } + // Record what matched. + apiFilter, err := c.FilterToAPIFilterV2(ctx, filter) + if err != nil { + return nil, err } - // We'll keep going after this in case there's a filter with a hide action that also matches. + filterResults = append(filterResults, apimodel.FilterResult{ + Filter: *apiFilter, + KeywordMatches: keywordMatches, + StatusMatches: statusMatches, + }) case gtsmodel.FilterActionHide: // Don't show this status. Immediate return. @@ -799,7 +792,7 @@ func (c *Converter) statusToAPIFilterResult( } } - return filterResult, nil + return filterResults, nil } // filterableTextFields returns all text from a status that we might want to filter on: @@ -1105,11 +1098,11 @@ func (c *Converter) statusToFrontend( } // Apply filters. - filterResult, err := c.statusToAPIFilterResult(ctx, s, requestingAccount, filterContext, filters) + filterResults, err := c.statusToAPIFilterResults(ctx, s, requestingAccount, filterContext, filters) if err != nil { return nil, fmt.Errorf("error applying filters: %w", err) } - apiStatus.Filtered = filterResult + apiStatus.Filtered = filterResults return apiStatus, nil } diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index a630b9c12f..ff58c531f2 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -661,30 +661,32 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { "card": null, "poll": null, "text": "hello world! #welcome ! first post on the instance :rainbow: ! fnord", - "filtered": { - "filter": { - "id": "01HN26VM6KZTW1ANNRVSBMA461", - "title": "fnord", - "context": [ - "home", - "public" - ], - "expires_at": null, - "filter_action": "warn", - "keywords": [ - { - "id": "01HN272TAVWAXX72ZX4M8JZ0PS", - "keyword": "fnord", - "whole_word": true - } + "filtered": [ + { + "filter": { + "id": "01HN26VM6KZTW1ANNRVSBMA461", + "title": "fnord", + "context": [ + "home", + "public" + ], + "expires_at": null, + "filter_action": "warn", + "keywords": [ + { + "id": "01HN272TAVWAXX72ZX4M8JZ0PS", + "keyword": "fnord", + "whole_word": true + } + ], + "statuses": [] + }, + "keyword_matches": [ + "fnord" ], - "statuses": [] - }, - "keyword_matches": [ - "fnord" - ], - "status_matches": [] - } + "status_matches": [] + } + ] }`, string(b)) } From 36a156cc132ab87682ca811afed6e96caa0ab16a Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 18:25:11 -0700 Subject: [PATCH 04/11] Make matching case-insensitive --- internal/typeutils/internaltofrontend.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 91a8d084da..03fbdf3c95 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -740,20 +740,19 @@ func (c *Converter) statusToAPIFilterResults( fields := filterableTextFields(s) for _, filterKeyword := range filter.Keywords { wholeWord := util.PtrValueOr(filterKeyword.WholeWord, false) + wordBreak := `` + if wholeWord { + wordBreak = `\b` + } + re, err := regexp.Compile(`(?i)` + wordBreak + regexp.QuoteMeta(filterKeyword.Keyword) + wordBreak) + if err != nil { + return nil, err + } var isMatch bool for _, field := range fields { - if wholeWord { - regexpIsMatch, err := regexp.MatchString(`\b`+regexp.QuoteMeta(filterKeyword.Keyword)+`\b`, field) - if err != nil { - return nil, err - } - if regexpIsMatch { - isMatch = true - } - } else { - if strings.Contains(field, filterKeyword.Keyword) { - isMatch = true - } + if re.MatchString(field) { + isMatch = true + break } } if isMatch { From fbd5e44d8fd5ff7163309fe828f769c80f0ae337 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 23:02:43 -0700 Subject: [PATCH 05/11] Remove TODOs that don't need to be done now --- internal/typeutils/internaltofrontend.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 03fbdf3c95..0adb0790b3 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -734,8 +734,6 @@ func (c *Converter) statusToAPIFilterResults( } // List all matching keywords. - // TODO: (Vyr) this might be sped up slightly by concatenating regexps in a cache somewhere, - // although we still have to keep track of which actual keywords matched. keywordMatches := make([]string, 0, len(filter.Keywords)) fields := filterableTextFields(s) for _, filterKeyword := range filter.Keywords { @@ -761,7 +759,6 @@ func (c *Converter) statusToAPIFilterResults( } // A status has only one ID. Not clear why this is a list in the Mastodon API. - // TODO: (Vyr) Likewise, there could be a set of status IDs in a cache. statusMatches := make([]string, 0, 1) for _, filterStatus := range filter.Statuses { if s.ID == filterStatus.StatusID { @@ -855,8 +852,7 @@ func (c *Converter) StatusToWebStatus( s *gtsmodel.Status, requestingAccount *gtsmodel.Account, ) (*apimodel.Status, error) { - // TODO: (Vyr) it's not clear to me why we'd have an account when requesting a web status - webStatus, err := c.statusToFrontend(ctx, s, requestingAccount, "", nil) + webStatus, err := c.statusToFrontend(ctx, s, requestingAccount, custom.FilterContextNone, nil) if err != nil { return nil, err } From 5f03f630aaf3a94a8ec962106b45367f0038e702 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 23:12:51 -0700 Subject: [PATCH 06/11] Add missing filter check for notification --- internal/processing/workers/surfacenotify.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/processing/workers/surfacenotify.go b/internal/processing/workers/surfacenotify.go index dff687235d..f098b3c4dd 100644 --- a/internal/processing/workers/surfacenotify.go +++ b/internal/processing/workers/surfacenotify.go @@ -389,8 +389,13 @@ func (s *surface) notify( return gtserror.Newf("error putting notification in database: %w", err) } + filters, err := s.state.DB.GetFiltersForAccountID(ctx, targetAccount.ID) + if err != nil { + return gtserror.Newf("couldn't retrieve filters for account %s: %w", targetAccount.ID, err) + } + // Stream notification to the user. - apiNotif, err := s.converter.NotificationToAPINotification(ctx, notif, nil) + apiNotif, err := s.converter.NotificationToAPINotification(ctx, notif, filters) if err != nil { return gtserror.Newf("error converting notification to api representation: %w", err) } From 6f37930963aee4959102ad75853cf6b3bf3fbcb8 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 23:16:46 -0700 Subject: [PATCH 07/11] lint: rename ErrHideStatus --- internal/filter/custom/user.go | 4 ++-- internal/processing/timeline/public.go | 2 +- internal/processing/timeline/tag.go | 2 +- internal/processing/workers/surfacetimeline.go | 2 +- internal/timeline/prepare.go | 2 +- internal/typeutils/internaltofrontend.go | 8 ++++---- internal/typeutils/internaltofrontend_test.go | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/filter/custom/user.go b/internal/filter/custom/user.go index 3b0eaa299f..5b47b8519b 100644 --- a/internal/filter/custom/user.go +++ b/internal/filter/custom/user.go @@ -22,8 +22,8 @@ import ( "errors" ) -// HideStatus indicates that a status has been filtered and should not be returned at all. -var HideStatus = errors.New("hide status") +// ErrHideStatus indicates that a status has been filtered and should not be returned at all. +var ErrHideStatus = errors.New("hide status") // FilterContext determines the filters that apply to a given status or list of statuses. type FilterContext string diff --git a/internal/processing/timeline/public.go b/internal/processing/timeline/public.go index 12c4c7958a..9c4d2ea90a 100644 --- a/internal/processing/timeline/public.go +++ b/internal/processing/timeline/public.go @@ -99,7 +99,7 @@ outer: } apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester, custom.FilterContextPublic, filters) - if errors.Is(err, custom.HideStatus) { + if errors.Is(err, custom.ErrHideStatus) { continue } if err != nil { diff --git a/internal/processing/timeline/tag.go b/internal/processing/timeline/tag.go index 0d583a1e92..5046b74b92 100644 --- a/internal/processing/timeline/tag.go +++ b/internal/processing/timeline/tag.go @@ -130,7 +130,7 @@ func (p *Processor) packageTagResponse( } apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct, custom.FilterContextPublic, filters) - if errors.Is(err, custom.HideStatus) { + if errors.Is(err, custom.ErrHideStatus) { continue } if err != nil { diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index a563bed12f..c5aa1ede6b 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -567,7 +567,7 @@ func (s *surface) timelineStreamStatusUpdate( filters []*gtsmodel.Filter, ) error { apiStatus, err := s.converter.StatusToAPIStatus(ctx, status, account, custom.FilterContextHome, filters) - if errors.Is(err, custom.HideStatus) { + if errors.Is(err, custom.ErrHideStatus) { // Don't put this status in the stream. return nil } diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go index 769c94144c..affc0db92f 100644 --- a/internal/timeline/prepare.go +++ b/internal/timeline/prepare.go @@ -122,7 +122,7 @@ func (t *timeline) prepareXBetweenIDs(ctx context.Context, amount int, behindID for e, entry := range toPrepare { prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) if err != nil { - if errors.Is(err, custom.HideStatus) { + if errors.Is(err, custom.ErrHideStatus) { // This item has been filtered out by the requesting user's filters. // Remove it and skip past it. t.items.data.Remove(e) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 0adb0790b3..8bc8c6a093 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -682,7 +682,7 @@ func (c *Converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistor // // Filter context can be the empty string if these statuses are not being filtered. // -// If there is a matching "hide" filter, the returned status will be nil with a HideStatus error; +// If there is a matching "hide" filter, the returned status will be nil with a ErrHideStatus error; // callers need to handle that case by excluding it from results. func (c *Converter) StatusToAPIStatus( ctx context.Context, @@ -708,7 +708,7 @@ func (c *Converter) StatusToAPIStatus( // statusToAPIFilterResults applies filters to a status and returns an API filter result object. // The result may be nil if no filters matched. -// If the status should not be returned at all, it returns the HideStatus error. +// If the status should not be returned at all, it returns the ErrHideStatus error. func (c *Converter) statusToAPIFilterResults( ctx context.Context, s *gtsmodel.Status, @@ -783,7 +783,7 @@ func (c *Converter) statusToAPIFilterResults( case gtsmodel.FilterActionHide: // Don't show this status. Immediate return. - return nil, custom.HideStatus + return nil, custom.ErrHideStatus } } } @@ -1053,7 +1053,7 @@ func (c *Converter) statusToFrontend( if s.BoostOf != nil { reblog, err := c.StatusToAPIStatus(ctx, s.BoostOf, requestingAccount, filterContext, filters) - if errors.Is(err, custom.HideStatus) { + if errors.Is(err, custom.ErrHideStatus) { // If we'd hide the original status, hide the boost. return nil, err } diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index ff58c531f2..05cbcf1a1e 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -690,7 +690,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { }`, string(b)) } -// Test that a status which is filtered with a hide filter by the requesting user results in the HideStatus error. +// Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error. func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() { testStatus := suite.testStatuses["admin_account_status_1"] testStatus.Content += " fnord" @@ -709,7 +709,7 @@ func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() { custom.FilterContextHome, requestingAccountFilters, ) - suite.ErrorIs(err, custom.HideStatus) + suite.ErrorIs(err, custom.ErrHideStatus) } func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments() { From 7ebbbb4063c552d5002b231777c0951c96928728 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 31 Mar 2024 23:44:18 -0700 Subject: [PATCH 08/11] APIFilterActionToFilterAction not used yet --- internal/typeutils/frontendtointernal.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/internal/typeutils/frontendtointernal.go b/internal/typeutils/frontendtointernal.go index dc8174a203..3bb0933f32 100644 --- a/internal/typeutils/frontendtointernal.go +++ b/internal/typeutils/frontendtointernal.go @@ -47,13 +47,3 @@ func APIMarkerNameToMarkerName(m apimodel.MarkerName) gtsmodel.MarkerName { } return "" } - -func APIFilterActionToFilterAction(m apimodel.FilterAction) gtsmodel.FilterAction { - switch m { - case apimodel.FilterActionWarn: - return gtsmodel.FilterActionWarn - case apimodel.FilterActionHide: - return gtsmodel.FilterActionHide - } - return "" -} From 3f6ecd1c8b11428aeafaec46296fb86aa295dc75 Mon Sep 17 00:00:00 2001 From: tobi Date: Thu, 2 May 2024 16:35:55 +0200 Subject: [PATCH 09/11] swaggerino docseroni --- docs/api/swagger.yaml | 116 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index dda090c524..8bd43ae8e1 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -1,5 +1,9 @@ basePath: / definitions: + FilterAction: + title: FilterAction is the action to apply to statuses matching a filter. + type: string + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model InstanceConfigurationEmojis: properties: emoji_size_limit: @@ -1037,6 +1041,60 @@ definitions: type: string x-go-name: FilterContext x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + filterKeyword: + properties: + id: + description: The ID of the filter keyword entry in the database. + type: string + x-go-name: ID + keyword: + description: The text to be filtered. + example: fnord + type: string + x-go-name: Keyword + whole_word: + description: Should the filter consider word boundaries? + example: true + type: boolean + x-go-name: WholeWord + title: FilterKeyword represents text to filter within a v2 filter. + type: object + x-go-name: FilterKeyword + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + filterResult: + properties: + filter: + $ref: '#/definitions/filterV2' + keyword_matches: + description: The keywords within the filter that were matched. + items: + type: string + type: array + x-go-name: KeywordMatches + status_matches: + description: The status IDs within the filter that were matched. + items: + type: string + type: array + x-go-name: StatusMatches + title: FilterResult is returned along with a filtered status to explain why it was filtered. + type: object + x-go-name: FilterResult + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + filterStatus: + properties: + id: + description: The ID of the filter status entry in the database. + type: string + x-go-name: ID + phrase: + description: The status ID to be filtered. + type: string + x-go-name: StatusID + title: FilterStatus represents a single status to filter within a v2 filter. + type: object + x-go-name: FilterStatus + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model filterV1: description: |- Note that v1 filters are mapped to v2 filters and v2 filter keywords internally. @@ -1086,6 +1144,52 @@ definitions: type: object x-go-name: FilterV1 x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + filterV2: + description: v2 filters have names and can include multiple phrases and status IDs to filter. + properties: + context: + description: The contexts in which the filter should be applied. + example: + - home + - public + items: + $ref: '#/definitions/filterContext' + minItems: 1 + type: array + uniqueItems: true + x-go-name: Context + expires_at: + description: When the filter should no longer be applied. Null if the filter does not expire. + example: "2024-02-01T02:57:49Z" + type: string + x-go-name: ExpiresAt + filter_action: + $ref: '#/definitions/FilterAction' + id: + description: The ID of the filter in the database. + type: string + x-go-name: ID + keywords: + description: The keywords grouped under this filter. + items: + $ref: '#/definitions/filterKeyword' + type: array + x-go-name: Keywords + statuses: + description: The statuses grouped under this filter. + items: + $ref: '#/definitions/filterStatus' + type: array + x-go-name: Statuses + title: + description: The name of the filter. + example: Linux Words + type: string + x-go-name: Title + title: FilterV2 represents a user-defined filter for determining which statuses should not be shown to the user. + type: object + x-go-name: FilterV2 + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model headerFilter: properties: created_at: @@ -2118,6 +2222,12 @@ definitions: format: int64 type: integer x-go-name: FavouritesCount + filtered: + description: A list of filters that matched this status and why they matched, if there are any such filters. + items: + $ref: '#/definitions/filterResult' + type: array + x-go-name: Filtered id: description: ID of the status. example: 01FBVD42CQ3ZEEVMW180SBX03B @@ -2321,6 +2431,12 @@ definitions: format: int64 type: integer x-go-name: FavouritesCount + filtered: + description: A list of filters that matched this status and why they matched, if there are any such filters. + items: + $ref: '#/definitions/filterResult' + type: array + x-go-name: Filtered id: description: ID of the status. example: 01FBVD42CQ3ZEEVMW180SBX03B From ef20a4e9a1bcb6f842c8b7e98e802fb72044d783 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 2 May 2024 08:57:43 -0700 Subject: [PATCH 10/11] Address review comments --- internal/api/model/filterv2.go | 2 -- .../{custom/user.go => status/status.go} | 4 +-- internal/processing/account/bookmarks.go | 4 +-- internal/processing/account/statuses.go | 4 +-- internal/processing/common/status.go | 3 +- internal/processing/search/util.go | 3 +- internal/processing/status/get.go | 4 +-- .../processing/stream/statusupdate_test.go | 4 +-- internal/processing/timeline/faved.go | 4 +-- internal/processing/timeline/home.go | 4 +-- internal/processing/timeline/list.go | 4 +-- internal/processing/timeline/public.go | 6 ++-- internal/processing/timeline/tag.go | 6 ++-- .../processing/workers/fromclientapi_test.go | 4 +-- .../processing/workers/surfacetimeline.go | 8 ++--- internal/timeline/prepare.go | 4 +-- internal/typeutils/internaltofrontend.go | 34 +++++++++---------- internal/typeutils/internaltofrontend_test.go | 14 ++++---- 18 files changed, 58 insertions(+), 58 deletions(-) rename internal/filter/{custom/user.go => status/status.go} (96%) diff --git a/internal/api/model/filterv2.go b/internal/api/model/filterv2.go index cf7a129c36..cacb197ea5 100644 --- a/internal/api/model/filterv2.go +++ b/internal/api/model/filterv2.go @@ -67,8 +67,6 @@ const ( FilterActionWarn FilterAction = "warn" // FilterActionHide filters will remove this status from API results. FilterActionHide FilterAction = "hide" - - FilterActionNumValues = 2 ) // FilterKeyword represents text to filter within a v2 filter. diff --git a/internal/filter/custom/user.go b/internal/filter/status/status.go similarity index 96% rename from internal/filter/custom/user.go rename to internal/filter/status/status.go index 5b47b8519b..7cf0a7a1e3 100644 --- a/internal/filter/custom/user.go +++ b/internal/filter/status/status.go @@ -15,8 +15,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -// Package custom represents custom filters managed by the user through the API. -package custom +// Package status represents status filters managed by the user through the API. +package status import ( "errors" diff --git a/internal/processing/account/bookmarks.go b/internal/processing/account/bookmarks.go index d934c9b4e6..5618934ae2 100644 --- a/internal/processing/account/bookmarks.go +++ b/internal/processing/account/bookmarks.go @@ -23,7 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -75,7 +75,7 @@ func (p *Processor) BookmarksGet(ctx context.Context, requestingAccount *gtsmode } // Convert the status. - item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextNone, nil) + item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextNone, nil) if err != nil { log.Errorf(ctx, "error converting bookmarked status to api: %s", err) continue diff --git a/internal/processing/account/statuses.go b/internal/processing/account/statuses.go index 6cf519044a..8f05483713 100644 --- a/internal/processing/account/statuses.go +++ b/internal/processing/account/statuses.go @@ -24,7 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -105,7 +105,7 @@ func (p *Processor) StatusesGet( for _, s := range filtered { // Convert filtered statuses to API statuses. - item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, custom.FilterContextAccount, filters) + item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, statusfilter.FilterContextAccount, filters) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/common/status.go b/internal/processing/common/status.go index 7aa8363069..bb46ee38cc 100644 --- a/internal/processing/common/status.go +++ b/internal/processing/common/status.go @@ -24,6 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -184,7 +185,7 @@ func (p *Processor) GetAPIStatus( apiStatus *apimodel.Status, errWithCode gtserror.WithCode, ) { - apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester, "", nil) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, target, requester, statusfilter.FilterContextNone, nil) if err != nil { err = gtserror.Newf("error converting status: %w", err) return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/search/util.go b/internal/processing/search/util.go index 60714f95e1..196fef5fc8 100644 --- a/internal/processing/search/util.go +++ b/internal/processing/search/util.go @@ -21,6 +21,7 @@ import ( "context" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -113,7 +114,7 @@ func (p *Processor) packageStatuses( continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, "", nil) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextNone, nil) if err != nil { log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", status.ID, err) continue diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 5a68fc0a0b..c05f3effd9 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -23,7 +23,7 @@ import ( "strings" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -287,7 +287,7 @@ func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel. return nil, gtserror.NewErrorInternalError(err) } convert := func(ctx context.Context, status *gtsmodel.Status, requestingAccount *gtsmodel.Account) (*apimodel.Status, error) { - return p.converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextThread, filters) + return p.converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextThread, filters) } return p.contextGet(ctx, requestingAccount, targetStatusID, convert) } diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index e0417b0c92..12971caa1f 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -24,7 +24,7 @@ import ( "testing" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/stream" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -40,7 +40,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.NoError(errWithCode) editedStatus := suite.testStatuses["remote_account_1_status_1"] - apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account, custom.FilterContextNotifications, nil) + apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(context.Background(), editedStatus, account, statusfilter.FilterContextNotifications, nil) suite.NoError(err) suite.streamProcessor.StatusUpdate(context.Background(), account, apiStatus, stream.TimelineHome) diff --git a/internal/processing/timeline/faved.go b/internal/processing/timeline/faved.go index cf5c0c6143..c3b0e1837e 100644 --- a/internal/processing/timeline/faved.go +++ b/internal/processing/timeline/faved.go @@ -24,7 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/oauth" @@ -55,7 +55,7 @@ func (p *Processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, ma continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, custom.FilterContextNone, nil) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, statusfilter.FilterContextNone, nil) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/timeline/home.go b/internal/processing/timeline/home.go index 3b86f3555e..e174b34283 100644 --- a/internal/processing/timeline/home.go +++ b/internal/processing/timeline/home.go @@ -23,7 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -105,7 +105,7 @@ func HomeTimelineStatusPrepare(state *state.State, converter *typeutils.Converte return nil, err } - return converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextHome, filters) + return converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextHome, filters) } } diff --git a/internal/processing/timeline/list.go b/internal/processing/timeline/list.go index 4703bf941f..60cdbac7a3 100644 --- a/internal/processing/timeline/list.go +++ b/internal/processing/timeline/list.go @@ -23,7 +23,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -117,7 +117,7 @@ func ListTimelineStatusPrepare(state *state.State, converter *typeutils.Converte return nil, err } - return converter.StatusToAPIStatus(ctx, status, requestingAccount, custom.FilterContextHome, filters) + return converter.StatusToAPIStatus(ctx, status, requestingAccount, statusfilter.FilterContextHome, filters) } } diff --git a/internal/processing/timeline/public.go b/internal/processing/timeline/public.go index 9c4d2ea90a..a0e594629b 100644 --- a/internal/processing/timeline/public.go +++ b/internal/processing/timeline/public.go @@ -24,7 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -98,8 +98,8 @@ outer: continue inner } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester, custom.FilterContextPublic, filters) - if errors.Is(err, custom.ErrHideStatus) { + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requester, statusfilter.FilterContextPublic, filters) + if errors.Is(err, statusfilter.ErrHideStatus) { continue } if err != nil { diff --git a/internal/processing/timeline/tag.go b/internal/processing/timeline/tag.go index 5046b74b92..5308cac591 100644 --- a/internal/processing/timeline/tag.go +++ b/internal/processing/timeline/tag.go @@ -24,7 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -129,8 +129,8 @@ func (p *Processor) packageTagResponse( continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct, custom.FilterContextPublic, filters) - if errors.Is(err, custom.ErrHideStatus) { + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, requestingAcct, statusfilter.FilterContextPublic, filters) + if errors.Is(err, statusfilter.ErrHideStatus) { continue } if err != nil { diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index ae59741205..6a12ce0439 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -28,7 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/messages" @@ -155,7 +155,7 @@ func (suite *FromClientAPITestSuite) statusJSON( ctx, status, requestingAccount, - custom.FilterContextNone, + statusfilter.FilterContextNone, nil, ) if err != nil { diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index 5be9495e2c..32fdd66e27 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -22,7 +22,7 @@ import ( "errors" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -357,7 +357,7 @@ func (s *Surface) timelineStatus( apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account, - custom.FilterContextHome, + statusfilter.FilterContextHome, filters, ) if err != nil { @@ -571,8 +571,8 @@ func (s *Surface) timelineStreamStatusUpdate( streamType string, filters []*gtsmodel.Filter, ) error { - apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account, custom.FilterContextHome, filters) - if errors.Is(err, custom.ErrHideStatus) { + apiStatus, err := s.Converter.StatusToAPIStatus(ctx, status, account, statusfilter.FilterContextHome, filters) + if errors.Is(err, statusfilter.ErrHideStatus) { // Don't put this status in the stream. return nil } diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go index affc0db92f..ec595ce42a 100644 --- a/internal/timeline/prepare.go +++ b/internal/timeline/prepare.go @@ -24,7 +24,7 @@ import ( "codeberg.org/gruf/go-kv" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" ) @@ -122,7 +122,7 @@ func (t *timeline) prepareXBetweenIDs(ctx context.Context, amount int, behindID for e, entry := range toPrepare { prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) if err != nil { - if errors.Is(err, custom.ErrHideStatus) { + if errors.Is(err, statusfilter.ErrHideStatus) { // This item has been filtered out by the requesting user's filters. // Remove it and skip past it. t.items.data.Remove(e) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 084effecc6..70bc1f8437 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -30,12 +30,13 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/language" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -696,7 +697,7 @@ func (c *Converter) StatusToAPIStatus( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, - filterContext custom.FilterContext, + filterContext statusfilter.FilterContext, filters []*gtsmodel.Filter, ) (*apimodel.Status, error) { apiStatus, err := c.statusToFrontend(ctx, s, requestingAccount, filterContext, filters) @@ -721,7 +722,7 @@ func (c *Converter) statusToAPIFilterResults( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, - filterContext custom.FilterContext, + filterContext statusfilter.FilterContext, filters []*gtsmodel.Filter, ) ([]apimodel.FilterResult, error) { if filterContext == "" || len(filters) == 0 || s.AccountID == requestingAccount.ID { @@ -791,7 +792,7 @@ func (c *Converter) statusToAPIFilterResults( case gtsmodel.FilterActionHide: // Don't show this status. Immediate return. - return nil, custom.ErrHideStatus + return nil, statusfilter.ErrHideStatus } } } @@ -812,8 +813,7 @@ func filterableTextFields(s *gtsmodel.Status) []string { fields := make([]string, 0, fieldCount) if s.Content != "" { - // TODO: (Vyr) convert this HTML field to plain text before returning - fields = append(fields, s.Content) + fields = append(fields, text.SanitizeToPlaintext(s.Content)) } if s.ContentWarning != "" { fields = append(fields, s.ContentWarning) @@ -835,17 +835,17 @@ func filterableTextFields(s *gtsmodel.Status) []string { } // filterAppliesInContext returns whether a given filter applies in a given context. -func filterAppliesInContext(filter *gtsmodel.Filter, filterContext custom.FilterContext) bool { +func filterAppliesInContext(filter *gtsmodel.Filter, filterContext statusfilter.FilterContext) bool { switch filterContext { - case custom.FilterContextHome: + case statusfilter.FilterContextHome: return util.PtrValueOr(filter.ContextHome, false) - case custom.FilterContextNotifications: + case statusfilter.FilterContextNotifications: return util.PtrValueOr(filter.ContextNotifications, false) - case custom.FilterContextPublic: + case statusfilter.FilterContextPublic: return util.PtrValueOr(filter.ContextPublic, false) - case custom.FilterContextThread: + case statusfilter.FilterContextThread: return util.PtrValueOr(filter.ContextThread, false) - case custom.FilterContextAccount: + case statusfilter.FilterContextAccount: return util.PtrValueOr(filter.ContextAccount, false) } return false @@ -860,7 +860,7 @@ func (c *Converter) StatusToWebStatus( s *gtsmodel.Status, requestingAccount *gtsmodel.Account, ) (*apimodel.Status, error) { - webStatus, err := c.statusToFrontend(ctx, s, requestingAccount, custom.FilterContextNone, nil) + webStatus, err := c.statusToFrontend(ctx, s, requestingAccount, statusfilter.FilterContextNone, nil) if err != nil { return nil, err } @@ -962,7 +962,7 @@ func (c *Converter) statusToFrontend( ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account, - filterContext custom.FilterContext, + filterContext statusfilter.FilterContext, filters []*gtsmodel.Filter, ) (*apimodel.Status, error) { // Try to populate status struct pointer fields. @@ -1063,7 +1063,7 @@ func (c *Converter) statusToFrontend( if s.BoostOf != nil { reblog, err := c.StatusToAPIStatus(ctx, s.BoostOf, requestingAccount, filterContext, filters) - if errors.Is(err, custom.ErrHideStatus) { + if errors.Is(err, statusfilter.ErrHideStatus) { // If we'd hide the original status, hide the boost. return nil, err } @@ -1453,7 +1453,7 @@ func (c *Converter) NotificationToAPINotification(ctx context.Context, n *gtsmod } var err error - apiStatus, err = c.StatusToAPIStatus(ctx, n.Status, n.TargetAccount, custom.FilterContextNotifications, filters) + apiStatus, err = c.StatusToAPIStatus(ctx, n.Status, n.TargetAccount, statusfilter.FilterContextNotifications, filters) if err != nil { return nil, fmt.Errorf("NotificationToapi: error converting status to api: %s", err) } @@ -1606,7 +1606,7 @@ func (c *Converter) ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Repo } } for _, s := range r.Statuses { - status, err := c.StatusToAPIStatus(ctx, s, requestingAccount, "", nil) + status, err := c.StatusToAPIStatus(ctx, s, requestingAccount, statusfilter.FilterContextNone, nil) if err != nil { return nil, fmt.Errorf("ReportToAdminAPIReport: error converting status with id %s to api status: %w", s.ID, err) } diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 432e24dcc2..2c4f28a9b3 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/custom" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -428,7 +428,7 @@ func (suite *InternalToFrontendTestSuite) TestLocalInstanceAccountToFrontendBloc func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { testStatus := suite.testStatuses["admin_account_status_1"] requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, statusfilter.FilterContextNone, nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -553,7 +553,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { context.Background(), testStatus, requestingAccount, - custom.FilterContextHome, + statusfilter.FilterContextHome, requestingAccountFilters, ) suite.NoError(err) @@ -707,17 +707,17 @@ func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() { context.Background(), testStatus, requestingAccount, - custom.FilterContextHome, + statusfilter.FilterContextHome, requestingAccountFilters, ) - suite.ErrorIs(err, custom.ErrHideStatus) + suite.ErrorIs(err, statusfilter.ErrHideStatus) } func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments() { testStatus := suite.testStatuses["remote_account_2_status_1"] requestingAccount := suite.testAccounts["admin_account"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, statusfilter.FilterContextNone, nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -950,7 +950,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Language = "" requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, "", nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, statusfilter.FilterContextNone, nil) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") From 9ca82930ef5c598948c01b3c7b9284cdbc577c08 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 2 May 2024 09:09:44 -0700 Subject: [PATCH 11/11] Add apimodel.FilterActionNone --- internal/api/model/filterv2.go | 2 ++ internal/typeutils/internaltofrontend.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/api/model/filterv2.go b/internal/api/model/filterv2.go index cacb197ea5..797c97213f 100644 --- a/internal/api/model/filterv2.go +++ b/internal/api/model/filterv2.go @@ -63,6 +63,8 @@ type FilterV2 struct { type FilterAction string const ( + // FilterActionNone filters should not exist, except internally, for partially constructed or invalid filters. + FilterActionNone FilterAction = "" // FilterActionWarn filters will include this status in API results with a warning. FilterActionWarn FilterAction = "warn" // FilterActionHide filters will remove this status from API results. diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 70bc1f8437..7a55722676 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1922,7 +1922,7 @@ func filterActionToAPIFilterAction(m gtsmodel.FilterAction) apimodel.FilterActio case gtsmodel.FilterActionHide: return apimodel.FilterActionHide } - return "" + return apimodel.FilterActionNone } // convertEmojisToAPIEmojis will convert a slice of GTS model emojis to frontend API model emojis, falling back to IDs if no GTS models supplied.