[feature/frontend] Add player for audio files; use thumbnail for `pos…

…ter` (#3099)

* [feature/frontend] Audio player for audio media types

* use video preview images for previews instead of video itself

* don't preload

* update tests for new zork status

* collapse media gallery into single row when small

internal/api/activitypub/users/outboxget_test.go

@@ -82,7 +82,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
   "@context": "https://www.w3.org/ns/activitystreams",
   "first": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40",
   "id": "http://localhost:8080/users/the_mighty_zork/outbox",
-  "totalItems": 7,
+  "totalItems": 8,
   "type": "OrderedCollection"
 }`, dst.String())
 
@@ -161,7 +161,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
   ],
   "partOf": "http://localhost:8080/users/the_mighty_zork/outbox",
   "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01HH9KYNQPA416TNJ53NSATP40",
-  "totalItems": 7,
+  "totalItems": 8,
   "type": "OrderedCollectionPage"
 }`, dst.String())
 
@@ -224,7 +224,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
   "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY",
   "orderedItems": [],
   "partOf": "http://localhost:8080/users/the_mighty_zork/outbox",
-  "totalItems": 7,
+  "totalItems": 8,
   "type": "OrderedCollectionPage"
 }`, dst.String())
 

internal/api/client/accounts/accountverify_test.go

@@ -79,7 +79,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() {
 	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic)
 	suite.Equal(2, apimodelAccount.FollowersCount)
 	suite.Equal(2, apimodelAccount.FollowingCount)
-	suite.Equal(7, apimodelAccount.StatusesCount)
+	suite.Equal(8, apimodelAccount.StatusesCount)
 	suite.EqualValues(gtsmodel.VisibilityPublic, apimodelAccount.Source.Privacy)
 	suite.Equal(testAccount.Settings.Language, apimodelAccount.Source.Language)
 	suite.Equal(testAccount.NoteRaw, apimodelAccount.Source.Note)

internal/api/client/admin/accountsgetv2_test.go

@@ -240,8 +240,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() {
       "header_description": "A very old-school screenshot of the original team fortress mod for quake",
       "followers_count": 2,
       "following_count": 2,
-      "statuses_count": 7,
-      "last_status_at": "2023-12-10T09:24:00.000Z",
+      "statuses_count": 8,
+      "last_status_at": "2024-01-10T09:24:00.000Z",
       "emojis": [],
       "fields": [],
       "enable_rss": true,

internal/api/client/instance/instancepatch_test.go

@@ -135,7 +135,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/assets/logo.png",
@@ -256,7 +256,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/assets/logo.png",
@@ -377,7 +377,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/assets/logo.png",
@@ -549,7 +549,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/assets/logo.png",
@@ -692,7 +692,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/attachment/original/`+instanceAccount.AvatarMediaAttachment.ID+`.gif",`+`
@@ -850,7 +850,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
   },
   "stats": {
     "domain_count": 2,
-    "status_count": 19,
+    "status_count": 20,
     "user_count": 4
   },
   "thumbnail": "http://localhost:8080/assets/logo.png",

internal/api/client/search/searchget_test.go

@@ -916,7 +916,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() {
 	}
 
 	suite.Len(searchResult.Accounts, 5)
-	suite.Len(searchResult.Statuses, 6)
+	suite.Len(searchResult.Statuses, 7)
 	suite.Len(searchResult.Hashtags, 0)
 }
 
@@ -959,7 +959,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() {
 	}
 
 	suite.Len(searchResult.Accounts, 2)
-	suite.Len(searchResult.Statuses, 6)
+	suite.Len(searchResult.Statuses, 7)
 	suite.Len(searchResult.Hashtags, 0)
 }
 
@@ -1002,7 +1002,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() {
 	}
 
 	suite.Len(searchResult.Accounts, 0)
-	suite.Len(searchResult.Statuses, 6)
+	suite.Len(searchResult.Statuses, 7)
 	suite.Len(searchResult.Hashtags, 0)
 }
 

internal/api/client/statuses/statushistory_test.go

@@ -114,8 +114,8 @@ func (suite *StatusHistoryTestSuite) TestGetHistory() {
       "header_description": "A very old-school screenshot of the original team fortress mod for quake",
       "followers_count": 2,
       "following_count": 2,
-      "statuses_count": 7,
-      "last_status_at": "2023-12-10T09:24:00.000Z",
+      "statuses_count": 8,
+      "last_status_at": "2024-01-10T09:24:00.000Z",
       "emojis": [],
       "fields": [],
       "enable_rss": true,

internal/api/client/statuses/statusmute_test.go

@@ -132,8 +132,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() {
     "header_description": "A very old-school screenshot of the original team fortress mod for quake",
     "followers_count": 2,
     "following_count": 2,
-    "statuses_count": 7,
-    "last_status_at": "2023-12-10T09:24:00.000Z",
+    "statuses_count": 8,
+    "last_status_at": "2024-01-10T09:24:00.000Z",
     "emojis": [],
     "fields": [],
     "enable_rss": true,
@@ -197,8 +197,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() {
     "header_description": "A very old-school screenshot of the original team fortress mod for quake",
     "followers_count": 2,
     "following_count": 2,
-    "statuses_count": 7,
-    "last_status_at": "2023-12-10T09:24:00.000Z",
+    "statuses_count": 8,
+    "last_status_at": "2024-01-10T09:24:00.000Z",
     "emojis": [],
     "fields": [],
     "enable_rss": true,

internal/api/model/attachment.go

@@ -90,12 +90,23 @@ type Attachment struct {
 	// A hash computed by the BlurHash algorithm, for generating colorful preview thumbnails when media has not been downloaded yet.
 	// See https://github.com/woltapp/blurhash
 	Blurhash *string `json:"blurhash"`
+}
+
+// WebAttachment is like Attachment, but with
+// additional fields not exposed via JSON;
+// used only internally for templating etc.
+//
+// swagger:ignore
+type WebAttachment struct {
+	*Attachment
 
-	// Additional fields not exposed via JSON
-	// (used only internally for templating etc).
+	// Parent status of this
+	// media is sensitive.
+	Sensitive bool
 
-	// Parent status of this media is sensitive.
-	Sensitive bool `json:"-"`
+	// MIME type of
+	// the attachment.
+	MIMEType string
 }
 
 // MediaMeta models media metadata.

internal/api/model/status.go

@@ -111,6 +111,10 @@ type Status struct {
 type WebStatus struct {
 	*Status
 
+	// Web version of media
+	// attached to this status.
+	MediaAttachments []*WebAttachment `json:"media_attachments"`
+
 	// Template-ready language tag and
 	// string, based on *status.Language.
 	LanguageTag *language.Language

internal/db/bundb/account_test.go

@@ -46,7 +46,7 @@ type AccountTestSuite struct {
 func (suite *AccountTestSuite) TestGetAccountStatuses() {
 	statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, false, false, "", "", false, false)
 	suite.NoError(err)
-	suite.Len(statuses, 7)
+	suite.Len(statuses, 8)
 }
 
 func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() {
@@ -69,7 +69,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() {
 	if err != nil {
 		suite.FailNow(err.Error())
 	}
-	suite.Len(statuses, 1)
+	suite.Len(statuses, 2)
 
 	// try to get the last page (should be empty)
 	statuses, err = suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 3, false, false, statuses[len(statuses)-1].ID, "", false, false)
@@ -187,7 +187,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesExcludesSelfR
 func (suite *AccountTestSuite) TestGetAccountStatusesMediaOnly() {
 	statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, false, false, "", "", true, false)
 	suite.NoError(err)
-	suite.Len(statuses, 1)
+	suite.Len(statuses, 2)
 }
 
 func (suite *AccountTestSuite) TestGetAccountBy() {

internal/db/bundb/basic_test.go

@@ -114,7 +114,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
 	s := []*gtsmodel.Status{}
 	err := suite.db.GetAll(context.Background(), &s)
 	suite.NoError(err)
-	suite.Len(s, 23)
+	suite.Len(s, 24)
 }
 
 func (suite *BasicTestSuite) TestGetAllNotNull() {

internal/db/bundb/instance_test.go

@@ -47,7 +47,7 @@ func (suite *InstanceTestSuite) TestCountInstanceUsersRemote() {
 func (suite *InstanceTestSuite) TestCountInstanceStatuses() {
 	count, err := suite.db.CountInstanceStatuses(context.Background(), config.GetHost())
 	suite.NoError(err)
-	suite.Equal(19, count)
+	suite.Equal(20, count)
 }
 
 func (suite *InstanceTestSuite) TestCountInstanceStatusesRemote() {

internal/db/bundb/status_test.go

@@ -169,12 +169,7 @@ func (suite *StatusTestSuite) TestGetStatusChildren() {
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 	children, err := suite.db.GetStatusChildren(context.Background(), targetStatus.ID)
 	suite.NoError(err)
-	suite.Len(children, 2)
-	for _, c := range children {
-		suite.Equal(targetStatus.URI, c.InReplyToURI)
-		suite.Equal(targetStatus.AccountID, c.InReplyToAccountID)
-		suite.Equal(targetStatus.ID, c.InReplyToID)
-	}
+	suite.Len(children, 3)
 }
 
 func (suite *StatusTestSuite) TestDeleteStatus() {

internal/db/bundb/timeline_test.go

@@ -155,7 +155,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimeline() {
 		suite.FailNow(err.Error())
 	}
 
-	suite.checkStatuses(s, id.Highest, id.Lowest, 19)
+	suite.checkStatuses(s, id.Highest, id.Lowest, 20)
 }
 
 func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() {
@@ -187,7 +187,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() {
 		suite.FailNow(err.Error())
 	}
 
-	suite.checkStatuses(s, id.Highest, id.Lowest, 7)
+	suite.checkStatuses(s, id.Highest, id.Lowest, 8)
 }
 
 func (suite *TimelineTestSuite) TestGetHomeTimelineWithFutureStatus() {
@@ -209,7 +209,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineWithFutureStatus() {
 	}
 
 	suite.NotContains(s, futureStatus)
-	suite.checkStatuses(s, id.Highest, id.Lowest, 19)
+	suite.checkStatuses(s, id.Highest, id.Lowest, 20)
 }
 
 func (suite *TimelineTestSuite) TestGetHomeTimelineBackToFront() {
@@ -240,8 +240,8 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineFromHighest() {
 	}
 
 	suite.checkStatuses(s, id.Highest, id.Lowest, 5)
-	suite.Equal("01HH9KYNQPA416TNJ53NSATP40", s[0].ID)
-	suite.Equal("01G20ZM733MGN8J344T4ZDDFY1", s[len(s)-1].ID)
+	suite.Equal("01J2M1HPFSS54S60Y0KYV23KJE", s[0].ID)
+	suite.Equal("01G36SF3V6Y6V5BF9P4R7PQG7G", s[len(s)-1].ID)
 }
 
 func (suite *TimelineTestSuite) TestGetListTimelineNoParams() {

internal/media/ffmpeg.go

@@ -18,6 +18,7 @@
 package media
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"errors"
@@ -198,6 +199,30 @@ func (res *ffprobeResult) ImageMeta() (width int, height int, err error) {
 	return
 }
 
+// EmbeddedImageMeta extracts embedded image metadata contained within ffprobe'd media result
+// streams, should be used for pulling album image (can be animated image) from audio files.
+func (res *ffprobeResult) EmbeddedImageMeta() (width int, height int, framerate float32, err error) {
+	for _, stream := range res.Streams {
+		if stream.Width > width {
+			width = stream.Width
+		}
+		if stream.Height > height {
+			height = stream.Height
+		}
+		if fr := stream.GetFrameRate(); fr > 0 {
+			if framerate == 0 || fr < framerate {
+				framerate = fr
+			}
+		}
+	}
+	// Need width + height but
+	// no framerate is fine.
+	if width == 0 || height == 0 {
+		err = errors.New("invalid image stream(s)")
+	}
+	return
+}
+
 // VideoMeta extracts video metadata contained within ffprobe'd media result streams.
 func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) {
 	for _, stream := range res.Streams {
@@ -222,14 +247,15 @@ func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err
 type ffprobeStream struct {
 	CodecName    string `json:"codec_name"`
 	AvgFrameRate string `json:"avg_frame_rate"`
+	RFrameRate   string `json:"r_frame_rate"`
 	Width        int    `json:"width"`
 	Height       int    `json:"height"`
 	// + unused fields.
 }
 
 // GetFrameRate calculates float32 framerate value from stream json string.
 func (str *ffprobeStream) GetFrameRate() float32 {
-	if str.AvgFrameRate != "" {
+	numDen := func(strFR string) (float32, float32) {
 		var (
 			// numerator
 			num float32
@@ -239,7 +265,7 @@ func (str *ffprobeStream) GetFrameRate() float32 {
 		)
 
 		// Check for a provided inequality, i.e. numerator / denominator.
-		if p := strings.SplitN(str.AvgFrameRate, "/", 2); len(p) == 2 {
+		if p := strings.SplitN(strFR, "/", 2); len(p) == 2 {
 			n, _ := strconv.ParseFloat(p[0], 32)
 			d, _ := strconv.ParseFloat(p[1], 32)
 			num, den = float32(n), float32(d)
@@ -248,8 +274,26 @@ func (str *ffprobeStream) GetFrameRate() float32 {
 			num = float32(n)
 		}
 
-		return num / den
+		return num, den
+	}
+
+	var num, den float32
+	if str.AvgFrameRate != "" {
+		// Check if we have avg_frame_rate.
+		num, den = numDen(str.AvgFrameRate)
+	}
+
+	if num == 0 && str.RFrameRate != "" {
+		// Check if we have r_frame_rate.
+		num, den = numDen(str.RFrameRate)
 	}
+
+	if num != 0 {
+		// Found it.
+		// Avoid divide by zero.
+		return num / cmp.Or(den, 1)
+	}
+
 	return 0
 }