[feature] For video attachments, store + return fps, bitrate, duration

internal/db/bundb/migrations/20221220134514_mp4_jiggery_pokery.go

@@ -0,0 +1,59 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors [email protected]
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+	"strings"
+
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	up := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			_, err := tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_duration"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			_, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_framerate"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			_, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? INTEGER", bun.Ident("media_attachments"), bun.Ident("original_bitrate"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/federation/dereferencing/media_test.go

@@ -66,7 +66,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
 	suite.NotEmpty(attachment.ID)
 	suite.NotEmpty(attachment.CreatedAt)
 	suite.NotEmpty(attachment.UpdatedAt)
-	suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, attachment.FileMeta.Original.Size)
 	suite.Equal(1245, attachment.FileMeta.Original.Height)
 	suite.Equal(1664, attachment.FileMeta.Original.Width)
@@ -92,7 +92,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
 	suite.NotEmpty(dbAttachment.ID)
 	suite.NotEmpty(dbAttachment.CreatedAt)
 	suite.NotEmpty(dbAttachment.UpdatedAt)
-	suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, dbAttachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, dbAttachment.FileMeta.Original.Size)
 	suite.Equal(1245, dbAttachment.FileMeta.Original.Height)
 	suite.Equal(1664, dbAttachment.FileMeta.Original.Width)
@@ -147,7 +147,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
 	suite.NotEmpty(attachment.ID)
 	suite.NotEmpty(attachment.CreatedAt)
 	suite.NotEmpty(attachment.UpdatedAt)
-	suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, attachment.FileMeta.Original.Size)
 	suite.Equal(1245, attachment.FileMeta.Original.Height)
 	suite.Equal(1664, attachment.FileMeta.Original.Width)

internal/gtserror/multi.go

@@ -0,0 +1,45 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors [email protected]
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gtserror
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// MultiError allows encapsulating multiple errors under a singular instance,
+// which is useful when you only want to log on errors, not return early / bubble up.
+type MultiError []string
+
+func (e *MultiError) Append(err error) {
+	*e = append(*e, err.Error())
+}
+
+func (e *MultiError) Appendf(format string, args ...any) {
+	*e = append(*e, fmt.Sprintf(format, args...))
+}
+
+// Combine converts this multiError to a singular error instance, returning nil if empty.
+func (e MultiError) Combine() error {
+	if len(e) == 0 {
+		return nil
+	}
+	return errors.New(`"` + strings.Join(e, `","`) + `"`)
+}

internal/gtsmodel/mediaattachment.go

@@ -99,15 +99,18 @@ type Small struct {
 	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
 	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
 	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
-	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+	Aspect float32 `validate:"required_with=Width Height Size"`   // aspect ratio (width / height)
 }
 
 // Original can be used for original metadata for any media type
 type Original struct {
-	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
-	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
-	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
-	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+	Width     int      `validate:"required_with=Height Size Aspect"`  // width in pixels
+	Height    int      `validate:"required_with=Width Size Aspect"`   // height in pixels
+	Size      int      `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
+	Aspect    float32  `validate:"required_with=Width Height Size"`   // aspect ratio (width / height)
+	Duration  *float32 `validate:"-"`                                 // video-specific: duration of the video in seconds
+	Framerate *float32 `validate:"-"`                                 // video-specific: fps
+	Bitrate   *uint64  `validate:"-"`                                 // video-specific: bitrate
 }
 
 // Focus describes the 'center' of the image for display purposes.

internal/media/image.go

@@ -48,7 +48,7 @@ func decodeGif(r io.Reader) (*mediaMeta, error) {
 	width := gif.Config.Width
 	height := gif.Config.Height
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -85,7 +85,7 @@ func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
 	width := i.Bounds().Size().X
 	height := i.Bounds().Size().Y
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -167,7 +167,7 @@ func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bo
 	thumbX := thumb.Bounds().Size().X
 	thumbY := thumb.Bounds().Size().Y
 	size := thumbX * thumbY
-	aspect := float64(thumbX) / float64(thumbY)
+	aspect := float32(thumbX) / float32(thumbY)
 
 	im := &mediaMeta{
 		width:  thumbX,

internal/media/manager_test.go

@@ -407,9 +407,13 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	suite.Equal(accountID, attachment.AccountID)
 
 	// file meta should be correctly derived from the video
-	suite.EqualValues(gtsmodel.Original{
-		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
-	}, attachment.FileMeta.Original)
+	suite.Equal(338, attachment.FileMeta.Original.Width)
+	suite.Equal(240, attachment.FileMeta.Original.Height)
+	suite.Equal(81120, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.4083333, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(6.5862, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(29.000029, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0x3b3e1, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
 		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
 	}, attachment.FileMeta.Small)
@@ -440,14 +444,116 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
 	suite.NoError(err)
 	suite.NotEmpty(processedThumbnailBytes)
-	
+
 	processedThumbnailBytesExpected, err := os.ReadFile("./test/test-mp4-thumbnail.jpg")
 	suite.NoError(err)
 	suite.NotEmpty(processedThumbnailBytesExpected)
 
 	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
 }
 
+func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() {
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/longer-mp4-original.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// process the media with no additional info provided
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+	// fetch the attachment id from the processing media
+	attachmentID := processingMedia.AttachmentID()
+
+	// do a blocking call to fetch the attachment
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.NoError(err)
+	suite.NotNil(attachment)
+
+	// make sure it's got the stuff set on it that we expect
+	// the attachment ID and accountID we expect
+	suite.Equal(attachmentID, attachment.ID)
+	suite.Equal(accountID, attachment.AccountID)
+
+	// file meta should be correctly derived from the video
+	suite.Equal(600, attachment.FileMeta.Original.Width)
+	suite.Equal(330, attachment.FileMeta.Original.Height)
+	suite.Equal(198000, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.8181819, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(16.6, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(10, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate)
+	suite.EqualValues(gtsmodel.Small{
+		Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819,
+	}, attachment.FileMeta.Small)
+	suite.Equal("video/mp4", attachment.File.ContentType)
+	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal(109549, attachment.File.FileSize)
+	suite.Equal("", attachment.Blurhash)
+
+	// now make sure the attachment is in the database
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
+	suite.NoError(err)
+	suite.NotNil(dbAttachment)
+
+	// make sure the processed file is in storage
+	processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytes)
+
+	// load the processed bytes from our test folder, to compare
+	processedFullBytesExpected, err := os.ReadFile("./test/longer-mp4-processed.mp4")
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytesExpected)
+
+	// the bytes in storage should be what we expected
+	suite.Equal(processedFullBytesExpected, processedFullBytes)
+
+	// now do the same for the thumbnail and make sure it's what we expected
+	processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytes)
+
+	processedThumbnailBytesExpected, err := os.ReadFile("./test/longer-mp4-thumbnail.jpg")
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytesExpected)
+
+	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
+}
+
+func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() {
+	// try to load an 'mp4' that's actually an mkv in disguise
+
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/not-an.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// pre processing should go fine but...
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+
+	// we should get an error while loading
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"")
+	suite.Nil(attachment)
+}
+
 func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
 	ctx := context.Background()
 

internal/media/processingmedia.go

@@ -249,16 +249,32 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
 		}
 
 		// set appropriate fields on the attachment based on the image we derived
+
+		// generic fields
+		p.attachment.File.UpdatedAt = time.Now()
 		p.attachment.FileMeta.Original = gtsmodel.Original{
 			Width:  decoded.width,
 			Height: decoded.height,
 			Size:   decoded.size,
 			Aspect: decoded.aspect,
 		}
-		p.attachment.File.UpdatedAt = time.Now()
-		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
+
+		// nullable fields
+		if decoded.duration != 0 {
+			i := decoded.duration
+			p.attachment.FileMeta.Original.Duration = &i
+		}
+		if decoded.framerate != 0 {
+			i := decoded.framerate
+			p.attachment.FileMeta.Original.Framerate = &i
+		}
+		if decoded.bitrate != 0 {
+			i := decoded.bitrate
+			p.attachment.FileMeta.Original.Bitrate = &i
+		}
 
 		// we're done processing the full-size image
+		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
 		atomic.StoreInt32(&p.fullSizeState, int32(complete))
 		log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
 		fallthrough

internal/media/types.go

@@ -137,7 +137,12 @@ type mediaMeta struct {
 	width    int
 	height   int
 	size     int
-	aspect   float64
+	aspect   float32
 	blurhash string
 	small    []byte
+
+	// video-specific properties
+	duration  float32
+	framerate float32
+	bitrate   uint64
 }