fix a bunch of media tests, move filesize checking to callers of medi…

…a manager for more flexibility

go.mod

@@ -13,7 +13,7 @@ require (
 	codeberg.org/gruf/go-errors/v2 v2.3.2
 	codeberg.org/gruf/go-fastcopy v1.1.2
 	codeberg.org/gruf/go-ffmpreg v0.2.2
-	codeberg.org/gruf/go-iotools v0.0.0-20240708125855-e9d3aa24b560
+	codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf
 	codeberg.org/gruf/go-kv v1.6.4
 	codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f
 	codeberg.org/gruf/go-logger/v2 v2.2.1

go.sum

@@ -54,8 +54,8 @@ codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q
 codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q=
 codeberg.org/gruf/go-ffmpreg v0.2.2 h1:K4I/7+BuzPLOVjL3hzTFdL8Z9wC0oRCK3xMKNVE86TE=
 codeberg.org/gruf/go-ffmpreg v0.2.2/go.mod h1:oPMfBkOK7xmR/teT/dKW6SeMFpRos9ceR/OuUrxBfcQ=
-codeberg.org/gruf/go-iotools v0.0.0-20240708125855-e9d3aa24b560 h1:ycFsr/lpSXFFa49BUxn1sOu1fiUH2SstHjxyUWhX9+k=
-codeberg.org/gruf/go-iotools v0.0.0-20240708125855-e9d3aa24b560/go.mod h1:zZAICsp5rY7+hxnws2V0ePrWxE0Z2Z/KXcN3p/RQCfk=
+codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf h1:84s/ii8N6lYlskZjHH+DG6jyia8w2mXMZlRwFn8Gs3A=
+codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf/go.mod h1:zZAICsp5rY7+hxnws2V0ePrWxE0Z2Z/KXcN3p/RQCfk=
 codeberg.org/gruf/go-kv v1.6.4 h1:3NZiW8HVdBM3kpOiLb7XfRiihnzZWMAixdCznguhILk=
 codeberg.org/gruf/go-kv v1.6.4/go.mod h1:O/YkSvKiS9XsRolM3rqCd9YJmND7dAXu9z+PrlYO4bc=
 codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f h1:Ss6Z+vygy+jOGhj96d/GwsYYDd22QmIcH74zM7/nQkw=

internal/api/client/instance/instancepatch.go

@@ -182,13 +182,6 @@ func validateInstanceUpdate(form *apimodel.InstanceSettingsUpdateRequest) error
 		return errors.New("empty form submitted")
 	}
 
-	if form.Avatar != nil {
-		maxImageSize := config.GetMediaImageMaxSize()
-		if size := form.Avatar.Size; size > int64(maxImageSize) {
-			return fmt.Errorf("file size limit exceeded: limit is %d bytes but desired instance avatar was %d bytes", maxImageSize, size)
-		}
-	}
-
 	if form.AvatarDescription != nil {
 		maxDescriptionChars := config.GetMediaDescriptionMaxChars()
 		if length := len([]rune(*form.AvatarDescription)); length > maxDescriptionChars {

internal/api/client/media/mediacreate.go

@@ -153,22 +153,9 @@ func validateCreateMedia(form *apimodel.AttachmentRequest) error {
 		return errors.New("no attachment given")
 	}
 
-	maxVideoSize := config.GetMediaVideoMaxSize()
-	maxImageSize := config.GetMediaImageMaxSize()
 	minDescriptionChars := config.GetMediaDescriptionMinChars()
 	maxDescriptionChars := config.GetMediaDescriptionMaxChars()
 
-	// a very superficial check to see if no size limits are exceeded
-	// we still don't actually know which media types we're dealing with but the other handlers will go into more detail there
-	maxSize := maxVideoSize
-	if maxImageSize > maxSize {
-		maxSize = maxImageSize
-	}
-
-	if form.File.Size > int64(maxSize) {
-		return fmt.Errorf("file size limit exceeded: limit is %d bytes but attachment was %d bytes", maxSize, form.File.Size)
-	}
-
 	if length := len([]rune(form.Description)); length > maxDescriptionChars {
 		return fmt.Errorf("image description length must be between %d and %d characters (inclusive), but provided image description was %d chars", minDescriptionChars, maxDescriptionChars, length)
 	}

internal/config/config.go

@@ -92,13 +92,13 @@ type Configuration struct {
 	AccountsAllowCustomCSS   bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."`
 	AccountsCustomCSSLength  int  `name:"accounts-custom-css-length" usage:"Maximum permitted length (characters) of custom CSS for accounts."`
 
-	MediaImageMaxSize        bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
-	MediaVideoMaxSize        bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
 	MediaDescriptionMinChars int           `name:"media-description-min-chars" usage:"Min required chars for an image description"`
 	MediaDescriptionMaxChars int           `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
 	MediaRemoteCacheDays     int           `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
 	MediaEmojiLocalMaxSize   bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
 	MediaEmojiRemoteMaxSize  bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
+	MediaLocalMaxSize        bytesize.Size `name:"media-local-max-size" usage:"Max size in bytes of media uploaded to this instance via API"`
+	MediaRemoteMaxSize       bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
 	MediaCleanupFrom         string        `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
 	MediaCleanupEvery        time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`
 

internal/config/defaults.go

@@ -71,11 +71,11 @@ var Defaults = Configuration{
 	AccountsAllowCustomCSS:   false,
 	AccountsCustomCSSLength:  10000,
 
-	MediaImageMaxSize:        10 * bytesize.MiB,
-	MediaVideoMaxSize:        40 * bytesize.MiB,
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 1500,
 	MediaRemoteCacheDays:     7,
+	MediaLocalMaxSize:        40 * bytesize.MiB,
+	MediaRemoteMaxSize:       40 * bytesize.MiB,
 	MediaEmojiLocalMaxSize:   50 * bytesize.KiB,
 	MediaEmojiRemoteMaxSize:  100 * bytesize.KiB,
 	MediaCleanupFrom:         "00:00",        // Midnight.

internal/config/flags.go

@@ -97,11 +97,11 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Bool(AccountsAllowCustomCSSFlag(), cfg.AccountsAllowCustomCSS, fieldtag("AccountsAllowCustomCSS", "usage"))
 
 		// Media
-		cmd.Flags().Uint64(MediaImageMaxSizeFlag(), uint64(cfg.MediaImageMaxSize), fieldtag("MediaImageMaxSize", "usage"))
-		cmd.Flags().Uint64(MediaVideoMaxSizeFlag(), uint64(cfg.MediaVideoMaxSize), fieldtag("MediaVideoMaxSize", "usage"))
 		cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
 		cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
 		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
+		cmd.Flags().Uint64(MediaLocalMaxSizeFlag(), uint64(cfg.MediaLocalMaxSize), fieldtag("MediaLocalMaxSize", "usage"))
+		cmd.Flags().Uint64(MediaRemoteMaxSizeFlag(), uint64(cfg.MediaRemoteMaxSize), fieldtag("MediaRemoteMaxSize", "usage"))
 		cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))
 		cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage"))
 		cmd.Flags().String(MediaCleanupFromFlag(), cfg.MediaCleanupFrom, fieldtag("MediaCleanupFrom", "usage"))

internal/config/helpers.gen.go

@@ -1075,56 +1075,6 @@ func GetAccountsCustomCSSLength() int { return global.GetAccountsCustomCSSLength
 // SetAccountsCustomCSSLength safely sets the value for global configuration 'AccountsCustomCSSLength' field
 func SetAccountsCustomCSSLength(v int) { global.SetAccountsCustomCSSLength(v) }
 
-// GetMediaImageMaxSize safely fetches the Configuration value for state's 'MediaImageMaxSize' field
-func (st *ConfigState) GetMediaImageMaxSize() (v bytesize.Size) {
-	st.mutex.RLock()
-	v = st.config.MediaImageMaxSize
-	st.mutex.RUnlock()
-	return
-}
-
-// SetMediaImageMaxSize safely sets the Configuration value for state's 'MediaImageMaxSize' field
-func (st *ConfigState) SetMediaImageMaxSize(v bytesize.Size) {
-	st.mutex.Lock()
-	defer st.mutex.Unlock()
-	st.config.MediaImageMaxSize = v
-	st.reloadToViper()
-}
-
-// MediaImageMaxSizeFlag returns the flag name for the 'MediaImageMaxSize' field
-func MediaImageMaxSizeFlag() string { return "media-image-max-size" }
-
-// GetMediaImageMaxSize safely fetches the value for global configuration 'MediaImageMaxSize' field
-func GetMediaImageMaxSize() bytesize.Size { return global.GetMediaImageMaxSize() }
-
-// SetMediaImageMaxSize safely sets the value for global configuration 'MediaImageMaxSize' field
-func SetMediaImageMaxSize(v bytesize.Size) { global.SetMediaImageMaxSize(v) }
-
-// GetMediaVideoMaxSize safely fetches the Configuration value for state's 'MediaVideoMaxSize' field
-func (st *ConfigState) GetMediaVideoMaxSize() (v bytesize.Size) {
-	st.mutex.RLock()
-	v = st.config.MediaVideoMaxSize
-	st.mutex.RUnlock()
-	return
-}
-
-// SetMediaVideoMaxSize safely sets the Configuration value for state's 'MediaVideoMaxSize' field
-func (st *ConfigState) SetMediaVideoMaxSize(v bytesize.Size) {
-	st.mutex.Lock()
-	defer st.mutex.Unlock()
-	st.config.MediaVideoMaxSize = v
-	st.reloadToViper()
-}
-
-// MediaVideoMaxSizeFlag returns the flag name for the 'MediaVideoMaxSize' field
-func MediaVideoMaxSizeFlag() string { return "media-video-max-size" }
-
-// GetMediaVideoMaxSize safely fetches the value for global configuration 'MediaVideoMaxSize' field
-func GetMediaVideoMaxSize() bytesize.Size { return global.GetMediaVideoMaxSize() }
-
-// SetMediaVideoMaxSize safely sets the value for global configuration 'MediaVideoMaxSize' field
-func SetMediaVideoMaxSize(v bytesize.Size) { global.SetMediaVideoMaxSize(v) }
-
 // GetMediaDescriptionMinChars safely fetches the Configuration value for state's 'MediaDescriptionMinChars' field
 func (st *ConfigState) GetMediaDescriptionMinChars() (v int) {
 	st.mutex.RLock()
@@ -1250,6 +1200,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem
 // SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
 func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
 
+// GetMediaLocalMaxSize safely fetches the Configuration value for state's 'MediaLocalMaxSize' field
+func (st *ConfigState) GetMediaLocalMaxSize() (v bytesize.Size) {
+	st.mutex.RLock()
+	v = st.config.MediaLocalMaxSize
+	st.mutex.RUnlock()
+	return
+}
+
+// SetMediaLocalMaxSize safely sets the Configuration value for state's 'MediaLocalMaxSize' field
+func (st *ConfigState) SetMediaLocalMaxSize(v bytesize.Size) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaLocalMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaLocalMaxSizeFlag returns the flag name for the 'MediaLocalMaxSize' field
+func MediaLocalMaxSizeFlag() string { return "media-local-max-size" }
+
+// GetMediaLocalMaxSize safely fetches the value for global configuration 'MediaLocalMaxSize' field
+func GetMediaLocalMaxSize() bytesize.Size { return global.GetMediaLocalMaxSize() }
+
+// SetMediaLocalMaxSize safely sets the value for global configuration 'MediaLocalMaxSize' field
+func SetMediaLocalMaxSize(v bytesize.Size) { global.SetMediaLocalMaxSize(v) }
+
+// GetMediaRemoteMaxSize safely fetches the Configuration value for state's 'MediaRemoteMaxSize' field
+func (st *ConfigState) GetMediaRemoteMaxSize() (v bytesize.Size) {
+	st.mutex.RLock()
+	v = st.config.MediaRemoteMaxSize
+	st.mutex.RUnlock()
+	return
+}
+
+// SetMediaRemoteMaxSize safely sets the Configuration value for state's 'MediaRemoteMaxSize' field
+func (st *ConfigState) SetMediaRemoteMaxSize(v bytesize.Size) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaRemoteMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaRemoteMaxSizeFlag returns the flag name for the 'MediaRemoteMaxSize' field
+func MediaRemoteMaxSizeFlag() string { return "media-remote-max-size" }
+
+// GetMediaRemoteMaxSize safely fetches the value for global configuration 'MediaRemoteMaxSize' field
+func GetMediaRemoteMaxSize() bytesize.Size { return global.GetMediaRemoteMaxSize() }
+
+// SetMediaRemoteMaxSize safely sets the value for global configuration 'MediaRemoteMaxSize' field
+func SetMediaRemoteMaxSize(v bytesize.Size) { global.SetMediaRemoteMaxSize(v) }
+
 // GetMediaCleanupFrom safely fetches the Configuration value for state's 'MediaCleanupFrom' field
 func (st *ConfigState) GetMediaCleanupFrom() (v string) {
 	st.mutex.RLock()

internal/federation/dereferencing/emoji.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net/url"
 
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -90,9 +91,12 @@ func (d *Dereferencer) GetEmoji(
 		return nil, err
 	}
 
+	// Get maximum supported remote emoji size.
+	maxsz := config.GetMediaEmojiRemoteMaxSize()
+
 	// Prepare data function to dereference remote emoji media.
-	data := func(context.Context) (io.ReadCloser, int64, error) {
-		return tsport.DereferenceMedia(ctx, url)
+	data := func(context.Context) (io.ReadCloser, error) {
+		return tsport.DereferenceMedia(ctx, url, int64(maxsz))
 	}
 
 	// Pass along for safe processing.
@@ -171,9 +175,12 @@ func (d *Dereferencer) RefreshEmoji(
 		return nil, err
 	}
 
+	// Get maximum supported remote emoji size.
+	maxsz := config.GetMediaEmojiRemoteMaxSize()
+
 	// Prepare data function to dereference remote emoji media.
-	data := func(context.Context) (io.ReadCloser, int64, error) {
-		return tsport.DereferenceMedia(ctx, url)
+	data := func(context.Context) (io.ReadCloser, error) {
+		return tsport.DereferenceMedia(ctx, url, int64(maxsz))
 	}
 
 	// Pass along for safe processing.

internal/federation/dereferencing/media.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/url"
 
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -69,12 +70,15 @@ func (d *Dereferencer) GetMedia(
 		return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
 	}
 
+	// Get maximum supported remote media size.
+	maxsz := config.GetMediaRemoteMaxSize()
+
 	// Start processing remote attachment at URL.
 	processing, err := d.mediaManager.CreateMedia(
 		ctx,
 		accountID,
-		func(ctx context.Context) (io.ReadCloser, int64, error) {
-			return tsport.DereferenceMedia(ctx, url)
+		func(ctx context.Context) (io.ReadCloser, error) {
+			return tsport.DereferenceMedia(ctx, url, int64(maxsz))
 		},
 		info,
 	)
@@ -163,11 +167,14 @@ func (d *Dereferencer) RefreshMedia(
 		return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
 	}
 
+	// Get maximum supported remote media size.
+	maxsz := config.GetMediaRemoteMaxSize()
+
 	// Start processing remote attachment recache.
 	processing := d.mediaManager.RecacheMedia(
 		media,
-		func(ctx context.Context) (io.ReadCloser, int64, error) {
-			return tsport.DereferenceMedia(ctx, url)
+		func(ctx context.Context) (io.ReadCloser, error) {
+			return tsport.DereferenceMedia(ctx, url, int64(maxsz))
 		},
 	)
 

internal/httpclient/client.go

@@ -31,7 +31,6 @@ import (
 	"strings"
 	"time"
 
-	"codeberg.org/gruf/go-bytesize"
 	"codeberg.org/gruf/go-cache/v3"
 	errorsv2 "codeberg.org/gruf/go-errors/v2"
 	"codeberg.org/gruf/go-iotools"
@@ -89,9 +88,6 @@ type Config struct {
 	// WriteBufferSize: see http.Transport{}.WriteBufferSize.
 	WriteBufferSize int
 
-	// MaxBodySize determines the maximum fetchable body size.
-	MaxBodySize int64
-
 	// Timeout: see http.Client{}.Timeout.
 	Timeout time.Duration
 
@@ -111,7 +107,6 @@ type Config struct {
 type Client struct {
 	client   http.Client
 	badHosts cache.TTLCache[string, struct{}]
-	bodyMax  int64
 	retries  uint
 }
 
@@ -137,11 +132,6 @@ func New(cfg Config) *Client {
 		cfg.MaxIdleConns = cfg.MaxOpenConnsPerHost * 10
 	}
 
-	if cfg.MaxBodySize <= 0 {
-		// By default set this to a reasonable 40MB.
-		cfg.MaxBodySize = int64(40 * bytesize.MiB)
-	}
-
 	// Protect the dialer
 	// with IP range sanitizer.
 	d.Control = (&Sanitizer{
@@ -151,7 +141,6 @@ func New(cfg Config) *Client {
 
 	// Prepare client fields.
 	c.client.Timeout = cfg.Timeout
-	c.bodyMax = cfg.MaxBodySize
 
 	// Prepare transport TLS config.
 	tlsClientConfig := &tls.Config{
@@ -377,31 +366,15 @@ func (c *Client) do(r *Request) (rsp *http.Response, retry bool, err error) {
 	rbody := (io.Reader)(rsp.Body)
 	cbody := (io.Closer)(rsp.Body)
 
-	var limit int64
-
-	if limit = rsp.ContentLength; limit < 0 {
-		// If unknown, use max as reader limit.
-		limit = c.bodyMax
-	}
-
-	// Don't trust them, limit body reads.
-	rbody = io.LimitReader(rbody, limit)
-
-	// Wrap closer to ensure entire body drained BEFORE close.
+	// Wrap closer to ensure body drained BEFORE close.
 	cbody = iotools.CloserAfterCallback(cbody, func() {
 		_, _ = discard.ReadFrom(rbody)
 	})
 
-	// Wrap body with limit.
-	rsp.Body = &struct {
-		io.Reader
-		io.Closer
-	}{rbody, cbody}
-
-	// Check response body not too large.
-	if rsp.ContentLength > c.bodyMax {
-		_ = rsp.Body.Close()
-		return nil, false, ErrBodyTooLarge
+	// Set the wrapped response body.
+	rsp.Body = &iotools.ReadCloserType{
+		Reader: rbody,
+		Closer: cbody,
 	}
 
 	return rsp, true, nil