[feature] do not uncache status / emoji media if attached status is b…

…ookmarked (#2956)

* do not uncache status / emoji media if attached status is bookmarked

* add status bookmark and bookmark IDs caches

* update status bookmark tests

* move IsStatusBookmarkedBy() to StatusBookmark{} interface, rely on cache

* fix envparsing.sh test

internal/cache/cache.go

@@ -85,6 +85,8 @@ func (c *Caches) Init() {
 	c.initPollVoteIDs()
 	c.initReport()
 	c.initStatus()
+	c.initStatusBookmark()
+	c.initStatusBookmarkIDs()
 	c.initStatusFave()
 	c.initStatusFaveIDs()
 	c.initTag()
@@ -101,7 +103,7 @@ func (c *Caches) Init() {
 func (c *Caches) Start() {
 	log.Infof(nil, "start: %p", c)
 
-	tryUntil("starting *gtsmodel.Webfinger cache", 5, func() bool {
+	tryUntil("starting webfinger cache", 5, func() bool {
 		return c.GTS.Webfinger.Start(5 * time.Minute)
 	})
 }
@@ -111,7 +113,7 @@ func (c *Caches) Start() {
 func (c *Caches) Stop() {
 	log.Infof(nil, "stop: %p", c)
 
-	tryUntil("stopping *gtsmodel.Webfinger cache", 5, c.GTS.Webfinger.Stop)
+	tryUntil("stopping webfinger cache", 5, c.GTS.Webfinger.Stop)
 }
 
 // Sweep will sweep all the available caches to ensure none
@@ -153,6 +155,8 @@ func (c *Caches) Sweep(threshold float64) {
 	c.GTS.PollVoteIDs.Trim(threshold)
 	c.GTS.Report.Trim(threshold)
 	c.GTS.Status.Trim(threshold)
+	c.GTS.StatusBookmark.Trim(threshold)
+	c.GTS.StatusBookmarkIDs.Trim(threshold)
 	c.GTS.StatusFave.Trim(threshold)
 	c.GTS.StatusFaveIDs.Trim(threshold)
 	c.GTS.Tag.Trim(threshold)

internal/cache/db.go

@@ -139,6 +139,12 @@ type GTSCaches struct {
 	// Status provides access to the gtsmodel Status database cache.
 	Status StructCache[*gtsmodel.Status]
 
+	// StatusBookmark ...
+	StatusBookmark StructCache[*gtsmodel.StatusBookmark]
+
+	// StatusBookmarkIDs ...
+	StatusBookmarkIDs SliceCache[string]
+
 	// StatusFave provides access to the gtsmodel StatusFave database cache.
 	StatusFave StructCache[*gtsmodel.StatusFave]
 
@@ -1102,6 +1108,54 @@ func (c *Caches) initStatus() {
 	})
 }
 
+func (c *Caches) initStatusBookmark() {
+	// Calculate maximum cache size.
+	cap := calculateResultCacheMax(
+		sizeofStatusBookmark(), // model in-mem size.
+		config.GetCacheStatusBookmarkMemRatio(),
+	)
+
+	log.Infof(nil, "cache size = %d", cap)
+
+	copyF := func(s1 *gtsmodel.StatusBookmark) *gtsmodel.StatusBookmark {
+		s2 := new(gtsmodel.StatusBookmark)
+		*s2 = *s1
+
+		// Don't include ptr fields that
+		// will be populated separately.
+		s2.Account = nil
+		s2.TargetAccount = nil
+		s2.Status = nil
+
+		return s2
+	}
+
+	c.GTS.StatusBookmark.Init(structr.CacheConfig[*gtsmodel.StatusBookmark]{
+		Indices: []structr.IndexConfig{
+			{Fields: "ID"},
+			{Fields: "AccountID,StatusID"},
+			{Fields: "AccountID", Multiple: true},
+			{Fields: "TargetAccountID", Multiple: true},
+			{Fields: "StatusID", Multiple: true},
+		},
+		MaxSize:    cap,
+		IgnoreErr:  ignoreErrors,
+		Copy:       copyF,
+		Invalidate: c.OnInvalidateStatusBookmark,
+	})
+}
+
+func (c *Caches) initStatusBookmarkIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheStatusBookmarkIDsMemRatio(),
+	)
+
+	log.Infof(nil, "cache size = %d", cap)
+
+	c.GTS.StatusBookmarkIDs.Init(0, cap)
+}
+
 func (c *Caches) initStatusFave() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(

internal/cache/invalidate.go

@@ -198,6 +198,11 @@ func (c *Caches) OnInvalidateStatus(status *gtsmodel.Status) {
 	}
 }
 
+func (c *Caches) OnInvalidateStatusBookmark(bookmark *gtsmodel.StatusBookmark) {
+	// Invalidate status bookmark ID list for this status.
+	c.GTS.StatusBookmarkIDs.Invalidate(bookmark.StatusID)
+}
+
 func (c *Caches) OnInvalidateStatusFave(fave *gtsmodel.StatusFave) {
 	// Invalidate status fave ID list for this status.
 	c.GTS.StatusFaveIDs.Invalidate(fave.StatusID)

internal/cache/size.go

@@ -201,6 +201,8 @@ func totalOfRatios() float64 {
 		config.GetCachePollVoteMemRatio() +
 		config.GetCacheReportMemRatio() +
 		config.GetCacheStatusMemRatio() +
+		config.GetCacheStatusBookmarkMemRatio() +
+		config.GetCacheStatusBookmarkIDsMemRatio() +
 		config.GetCacheStatusFaveMemRatio() +
 		config.GetCacheStatusFaveIDsMemRatio() +
 		config.GetCacheTagMemRatio() +
@@ -566,7 +568,7 @@ func sizeofReport() uintptr {
 
 func sizeofStatus() uintptr {
 	return uintptr(size.Of(&gtsmodel.Status{
-		ID:                       exampleURI,
+		ID:                       exampleID,
 		URI:                      exampleURI,
 		URL:                      exampleURI,
 		Content:                  exampleText,
@@ -599,6 +601,20 @@ func sizeofStatus() uintptr {
 	}))
 }
 
+func sizeofStatusBookmark() uintptr {
+	return uintptr(size.Of(&gtsmodel.StatusBookmark{
+		ID:              exampleID,
+		AccountID:       exampleID,
+		Account:         nil,
+		TargetAccountID: exampleID,
+		TargetAccount:   nil,
+		StatusID:        exampleID,
+		Status:          nil,
+		CreatedAt:       exampleTime,
+		UpdatedAt:       exampleTime,
+	}))
+}
+
 func sizeofStatusFave() uintptr {
 	return uintptr(size.Of(&gtsmodel.StatusFave{
 		ID:              exampleID,

internal/cleaner/emoji.go

@@ -32,9 +32,7 @@ import (
 
 // Emoji encompasses a set of
 // emoji cleanup / admin utils.
-type Emoji struct {
-	*Cleaner
-}
+type Emoji struct{ *Cleaner }
 
 // All will execute all cleaner.Emoji utilities synchronously, including output logging.
 // Context will be checked for `gtscontext.DryRun()` in order to actually perform the action.
@@ -381,10 +379,20 @@ func (e *Emoji) uncacheRemote(ctx context.Context, after time.Time, emoji *gtsmo
 	}
 
 	for _, status := range statuses {
+		// Check if recently used status.
 		if status.FetchedAt.After(after) {
 			l.Debug("skipping due to recently fetched status")
 			return false, nil
 		}
+
+		// Check whether status is bookmarked by active accounts.
+		bookmarked, err := e.state.DB.IsStatusBookmarked(ctx, status.ID)
+		if err != nil {
+			return false, err
+		} else if bookmarked {
+			l.Debug("skipping due to bookmarked status")
+			return false, nil
+		}
 	}
 
 	// This emoji is too old, uncache it.

internal/cleaner/media.go

@@ -35,9 +35,7 @@ import (
 
 // Media encompasses a set of
 // media cleanup / admin utils.
-type Media struct {
-	*Cleaner
-}
+type Media struct{ *Cleaner }
 
 // All will execute all cleaner.Media utilities synchronously, including output logging.
 // Context will be checked for `gtscontext.DryRun()` in order to actually perform the action.
@@ -475,9 +473,21 @@ func (m *Media) uncacheRemote(ctx context.Context, after time.Time, media *gtsmo
 			return false, nil
 		}
 
-		if status != nil && status.FetchedAt.After(after) {
-			l.Debug("skipping due to recently fetched status")
-			return false, nil
+		if status != nil {
+			// Check if recently used status.
+			if status.FetchedAt.After(after) {
+				l.Debug("skipping due to recently fetched status")
+				return false, nil
+			}
+
+			// Check whether status is bookmarked by active accounts.
+			bookmarked, err := m.state.DB.IsStatusBookmarked(ctx, status.ID)
+			if err != nil {
+				return false, err
+			} else if bookmarked {
+				l.Debug("skipping due to bookmarked status")
+				return false, nil
+			}
 		}
 	}
 

internal/config/config.go

@@ -191,48 +191,50 @@ type HTTPClientConfiguration struct {
 }
 
 type CacheConfiguration struct {
-	MemoryTarget             bytesize.Size `name:"memory-target"`
-	AccountMemRatio          float64       `name:"account-mem-ratio"`
-	AccountNoteMemRatio      float64       `name:"account-note-mem-ratio"`
-	AccountSettingsMemRatio  float64       `name:"account-settings-mem-ratio"`
-	AccountStatsMemRatio     float64       `name:"account-stats-mem-ratio"`
-	ApplicationMemRatio      float64       `name:"application-mem-ratio"`
-	BlockMemRatio            float64       `name:"block-mem-ratio"`
-	BlockIDsMemRatio         float64       `name:"block-mem-ratio"`
-	BoostOfIDsMemRatio       float64       `name:"boost-of-ids-mem-ratio"`
-	ClientMemRatio           float64       `name:"client-mem-ratio"`
-	EmojiMemRatio            float64       `name:"emoji-mem-ratio"`
-	EmojiCategoryMemRatio    float64       `name:"emoji-category-mem-ratio"`
-	FilterMemRatio           float64       `name:"filter-mem-ratio"`
-	FilterKeywordMemRatio    float64       `name:"filter-keyword-mem-ratio"`
-	FilterStatusMemRatio     float64       `name:"filter-status-mem-ratio"`
-	FollowMemRatio           float64       `name:"follow-mem-ratio"`
-	FollowIDsMemRatio        float64       `name:"follow-ids-mem-ratio"`
-	FollowRequestMemRatio    float64       `name:"follow-request-mem-ratio"`
-	FollowRequestIDsMemRatio float64       `name:"follow-request-ids-mem-ratio"`
-	InReplyToIDsMemRatio     float64       `name:"in-reply-to-ids-mem-ratio"`
-	InstanceMemRatio         float64       `name:"instance-mem-ratio"`
-	ListMemRatio             float64       `name:"list-mem-ratio"`
-	ListEntryMemRatio        float64       `name:"list-entry-mem-ratio"`
-	MarkerMemRatio           float64       `name:"marker-mem-ratio"`
-	MediaMemRatio            float64       `name:"media-mem-ratio"`
-	MentionMemRatio          float64       `name:"mention-mem-ratio"`
-	MoveMemRatio             float64       `name:"move-mem-ratio"`
-	NotificationMemRatio     float64       `name:"notification-mem-ratio"`
-	PollMemRatio             float64       `name:"poll-mem-ratio"`
-	PollVoteMemRatio         float64       `name:"poll-vote-mem-ratio"`
-	PollVoteIDsMemRatio      float64       `name:"poll-vote-ids-mem-ratio"`
-	ReportMemRatio           float64       `name:"report-mem-ratio"`
-	StatusMemRatio           float64       `name:"status-mem-ratio"`
-	StatusFaveMemRatio       float64       `name:"status-fave-mem-ratio"`
-	StatusFaveIDsMemRatio    float64       `name:"status-fave-ids-mem-ratio"`
-	TagMemRatio              float64       `name:"tag-mem-ratio"`
-	ThreadMuteMemRatio       float64       `name:"thread-mute-mem-ratio"`
-	TokenMemRatio            float64       `name:"token-mem-ratio"`
-	TombstoneMemRatio        float64       `name:"tombstone-mem-ratio"`
-	UserMemRatio             float64       `name:"user-mem-ratio"`
-	WebfingerMemRatio        float64       `name:"webfinger-mem-ratio"`
-	VisibilityMemRatio       float64       `name:"visibility-mem-ratio"`
+	MemoryTarget              bytesize.Size `name:"memory-target"`
+	AccountMemRatio           float64       `name:"account-mem-ratio"`
+	AccountNoteMemRatio       float64       `name:"account-note-mem-ratio"`
+	AccountSettingsMemRatio   float64       `name:"account-settings-mem-ratio"`
+	AccountStatsMemRatio      float64       `name:"account-stats-mem-ratio"`
+	ApplicationMemRatio       float64       `name:"application-mem-ratio"`
+	BlockMemRatio             float64       `name:"block-mem-ratio"`
+	BlockIDsMemRatio          float64       `name:"block-mem-ratio"`
+	BoostOfIDsMemRatio        float64       `name:"boost-of-ids-mem-ratio"`
+	ClientMemRatio            float64       `name:"client-mem-ratio"`
+	EmojiMemRatio             float64       `name:"emoji-mem-ratio"`
+	EmojiCategoryMemRatio     float64       `name:"emoji-category-mem-ratio"`
+	FilterMemRatio            float64       `name:"filter-mem-ratio"`
+	FilterKeywordMemRatio     float64       `name:"filter-keyword-mem-ratio"`
+	FilterStatusMemRatio      float64       `name:"filter-status-mem-ratio"`
+	FollowMemRatio            float64       `name:"follow-mem-ratio"`
+	FollowIDsMemRatio         float64       `name:"follow-ids-mem-ratio"`
+	FollowRequestMemRatio     float64       `name:"follow-request-mem-ratio"`
+	FollowRequestIDsMemRatio  float64       `name:"follow-request-ids-mem-ratio"`
+	InReplyToIDsMemRatio      float64       `name:"in-reply-to-ids-mem-ratio"`
+	InstanceMemRatio          float64       `name:"instance-mem-ratio"`
+	ListMemRatio              float64       `name:"list-mem-ratio"`
+	ListEntryMemRatio         float64       `name:"list-entry-mem-ratio"`
+	MarkerMemRatio            float64       `name:"marker-mem-ratio"`
+	MediaMemRatio             float64       `name:"media-mem-ratio"`
+	MentionMemRatio           float64       `name:"mention-mem-ratio"`
+	MoveMemRatio              float64       `name:"move-mem-ratio"`
+	NotificationMemRatio      float64       `name:"notification-mem-ratio"`
+	PollMemRatio              float64       `name:"poll-mem-ratio"`
+	PollVoteMemRatio          float64       `name:"poll-vote-mem-ratio"`
+	PollVoteIDsMemRatio       float64       `name:"poll-vote-ids-mem-ratio"`
+	ReportMemRatio            float64       `name:"report-mem-ratio"`
+	StatusMemRatio            float64       `name:"status-mem-ratio"`
+	StatusBookmarkMemRatio    float64       `name:"status-bookmark-mem-ratio"`
+	StatusBookmarkIDsMemRatio float64       `name:"status-bookmark-ids-mem-ratio"`
+	StatusFaveMemRatio        float64       `name:"status-fave-mem-ratio"`
+	StatusFaveIDsMemRatio     float64       `name:"status-fave-ids-mem-ratio"`
+	TagMemRatio               float64       `name:"tag-mem-ratio"`
+	ThreadMuteMemRatio        float64       `name:"thread-mute-mem-ratio"`
+	TokenMemRatio             float64       `name:"token-mem-ratio"`
+	TombstoneMemRatio         float64       `name:"tombstone-mem-ratio"`
+	UserMemRatio              float64       `name:"user-mem-ratio"`
+	WebfingerMemRatio         float64       `name:"webfinger-mem-ratio"`
+	VisibilityMemRatio        float64       `name:"visibility-mem-ratio"`
 }
 
 // MarshalMap will marshal current Configuration into a map structure (useful for JSON/TOML/YAML).