-
-
Notifications
You must be signed in to change notification settings - Fork 349
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[chore] media pipeline improvements (#3110)
* don't set emoji / media image paths on failed download, migrate FileType from string to integer * fix incorrect uses of util.PtrOr, fix returned frontend media * fix migration not setting arguments correctly in where clause * fix not providing default with not null column * whoops * ensure a default gets set for media attachment file type * remove the exclusive flag from writing files in disk storage * rename PtrOr -> PtrOrZero, and rename PtrValueOr -> PtrOrValue to match * slight wording changes * use singular / plural word forms (no parentheses), is better for screen readers * update testmodels with unknown media type to have unset file details, update attachment focus handling converting to frontend, update tests * store first instance in ffmpeg wasm pool, fill remaining with closed instances
- Loading branch information
1 parent
0aadc2d
commit 72ba566
Showing
29 changed files
with
665 additions
and
395 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
124 changes: 124 additions & 0 deletions
124
internal/db/bundb/migrations/20240715204203_media_pipeline_improvements.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| // GoToSocial | ||
| // Copyright (C) GoToSocial Authors [email protected] | ||
| // SPDX-License-Identifier: AGPL-3.0-or-later | ||
| // | ||
| // This program is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Affero General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Affero General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU Affero General Public License | ||
| // along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package migrations | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| old_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20240715204203_media_pipeline_improvements" | ||
| new_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" | ||
|
|
||
| "github.com/uptrace/bun" | ||
| ) | ||
|
|
||
| func init() { | ||
| up := func(ctx context.Context, db *bun.DB) error { | ||
| if err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { | ||
| if _, err := tx.NewAddColumn(). | ||
| Table("media_attachments"). | ||
| ColumnExpr("? INTEGER NOT NULL DEFAULT ?", bun.Ident("type_new"), 0). | ||
| Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for old, new := range map[old_gtsmodel.FileType]new_gtsmodel.FileType{ | ||
| old_gtsmodel.FileTypeAudio: new_gtsmodel.FileTypeAudio, | ||
| old_gtsmodel.FileTypeImage: new_gtsmodel.FileTypeImage, | ||
| old_gtsmodel.FileTypeGifv: new_gtsmodel.FileTypeImage, | ||
| old_gtsmodel.FileTypeVideo: new_gtsmodel.FileTypeVideo, | ||
| old_gtsmodel.FileTypeUnknown: new_gtsmodel.FileTypeUnknown, | ||
| } { | ||
| if _, err := tx.NewUpdate(). | ||
| Table("media_attachments"). | ||
| Where("? = ?", bun.Ident("type"), old). | ||
| Set("? = ?", bun.Ident("type_new"), new). | ||
| Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| if _, err := tx.NewDropColumn(). | ||
| Table("media_attachments"). | ||
| ColumnExpr("?", bun.Ident("type")). | ||
| Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if _, err := tx.NewRaw( | ||
| "ALTER TABLE ? RENAME COLUMN ? TO ?", | ||
| bun.Ident("media_attachments"), | ||
| bun.Ident("type_new"), | ||
| bun.Ident("type"), | ||
| ).Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| }); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Zero-out attachment data | ||
| // for "unknown" non-locally | ||
| // stored media attachments. | ||
| if _, err := db.NewUpdate(). | ||
| Table("media_attachments"). | ||
| Where("? = ?", bun.Ident("type"), new_gtsmodel.FileTypeUnknown). | ||
| Set("? = ?", bun.Ident("url"), ""). | ||
| Set("? = ?", bun.Ident("file_path"), ""). | ||
| Set("? = ?", bun.Ident("file_content_type"), ""). | ||
| Set("? = ?", bun.Ident("file_file_size"), 0). | ||
| Set("? = ?", bun.Ident("thumbnail_path"), ""). | ||
| Set("? = ?", bun.Ident("thumbnail_content_type"), ""). | ||
| Set("? = ?", bun.Ident("thumbnail_file_size"), 0). | ||
| Set("? = ?", bun.Ident("thumbnail_url"), ""). | ||
| Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Zero-out emoji data for | ||
| // non-locally stored emoji. | ||
| if _, err := db.NewUpdate(). | ||
| Table("emojis"). | ||
| WhereOr("? = ?", bun.Ident("image_url"), ""). | ||
| WhereOr("? = ?", bun.Ident("image_path"), ""). | ||
| Set("? = ?", bun.Ident("image_path"), ""). | ||
| Set("? = ?", bun.Ident("image_url"), ""). | ||
| Set("? = ?", bun.Ident("image_file_size"), 0). | ||
| Set("? = ?", bun.Ident("image_content_type"), ""). | ||
| Set("? = ?", bun.Ident("image_static_path"), ""). | ||
| Set("? = ?", bun.Ident("image_static_url"), ""). | ||
| Set("? = ?", bun.Ident("image_static_file_size"), 0). | ||
| Set("? = ?", bun.Ident("image_static_content_type"), ""). | ||
| Exec(ctx); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| down := func(ctx context.Context, db *bun.DB) error { | ||
| return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { | ||
| return nil | ||
| }) | ||
| } | ||
|
|
||
| if err := Migrations.Register(up, down); err != nil { | ||
| panic(err) | ||
| } | ||
| } |
65 changes: 65 additions & 0 deletions
65
internal/db/bundb/migrations/20240715204203_media_pipeline_improvements/emoji.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // GoToSocial | ||
| // Copyright (C) GoToSocial Authors [email protected] | ||
| // SPDX-License-Identifier: AGPL-3.0-or-later | ||
| // | ||
| // This program is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Affero General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Affero General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU Affero General Public License | ||
| // along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package gtsmodel | ||
|
|
||
| import "time" | ||
|
|
||
| // Emoji represents a custom emoji that's been uploaded through the admin UI or downloaded from a remote instance. | ||
| type Emoji struct { | ||
| ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database | ||
| CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created | ||
| UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated | ||
| Shortcode string `bun:",nullzero,notnull,unique:domainshortcode"` // String shortcode for this emoji -- the part that's between colons. This should be a-zA-Z_ eg., 'blob_hug' 'purple_heart' 'Gay_Otter' Must be unique with domain. | ||
| Domain string `bun:",nullzero,unique:domainshortcode"` // Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis. | ||
| ImageRemoteURL string `bun:",nullzero"` // Where can this emoji be retrieved remotely? Null for local emojis. | ||
| ImageStaticRemoteURL string `bun:",nullzero"` // Where can a static / non-animated version of this emoji be retrieved remotely? Null for local emojis. | ||
| ImageURL string `bun:",nullzero"` // Where can this emoji be retrieved from the local server? Null for remote emojis. | ||
| ImageStaticURL string `bun:",nullzero"` // Where can a static version of this emoji be retrieved from the local server? Null for remote emojis. | ||
| ImagePath string `bun:",notnull"` // Path of the emoji image in the server storage system. | ||
| ImageStaticPath string `bun:",notnull"` // Path of a static version of the emoji image in the server storage system | ||
| ImageContentType string `bun:",notnull"` // MIME content type of the emoji image | ||
| ImageStaticContentType string `bun:",notnull"` // MIME content type of the static version of the emoji image. | ||
| ImageFileSize int `bun:",notnull"` // Size of the emoji image file in bytes, for serving purposes. | ||
| ImageStaticFileSize int `bun:",notnull"` // Size of the static version of the emoji image file in bytes, for serving purposes. | ||
| Disabled *bool `bun:",nullzero,notnull,default:false"` // Has a moderation action disabled this emoji from being shown? | ||
| URI string `bun:",nullzero,notnull,unique"` // ActivityPub uri of this emoji. Something like 'https://example.org/emojis/1234' | ||
| VisibleInPicker *bool `bun:",nullzero,notnull,default:true"` // Is this emoji visible in the admin emoji picker? | ||
| Category *EmojiCategory `bun:"rel:belongs-to"` // In which emoji category is this emoji visible? | ||
| CategoryID string `bun:"type:CHAR(26),nullzero"` // ID of the category this emoji belongs to. | ||
| Cached *bool `bun:",nullzero,notnull,default:false"` // whether emoji is cached in locally in gotosocial storage. | ||
| } | ||
|
|
||
| // IsLocal returns true if the emoji is | ||
| // local to this instance., ie., it did | ||
| // not originate from a remote instance. | ||
| func (e *Emoji) IsLocal() bool { | ||
| return e.Domain == "" | ||
| } | ||
|
|
||
| // ShortcodeDomain returns the [shortcode]@[domain] for the given emoji. | ||
| func (e *Emoji) ShortcodeDomain() string { | ||
| return e.Shortcode + "@" + e.Domain | ||
| } | ||
|
|
||
| // EmojiCategory represents a grouping of custom emojis. | ||
| type EmojiCategory struct { | ||
| ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database | ||
| CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created | ||
| UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated | ||
| Name string `bun:",nullzero,notnull,unique"` // name of this category | ||
| } |
Oops, something went wrong.