Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix merging artifact chunks error when minio storage basepath is set #28555

Merged
merged 4 commits into from
Dec 21, 2023
Merged

Fix merging artifact chunks error when minio storage basepath is set #28555

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f2b702a
fix: trim minio basepath when merging chunks
fuxiaohei Dec 20, 2023
b70a3cc
fix: update artifact chunk basename to ignore directory value in stor…
fuxiaohei Dec 20, 2023
3644975
update filepath to handle file's path from storage
fuxiaohei Dec 20, 2023
2e72950
Merge branch 'main' into fix/artifact-upload-minio
GiteaBot Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions routers/api/actions/artifacts_chunks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ import (
"io"
"path/filepath"
"sort"
"strings"
"time"

"code.gitea.io/gitea/models/actions"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
)

Expand All @@ -26,6 +28,7 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
contentRange := ctx.Req.Header.Get("Content-Range")
start, end, length := int64(0), int64(0), int64(0)
if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil {
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
return -1, fmt.Errorf("parse content range error: %v", err)
}
// build chunk store path
Expand Down Expand Up @@ -64,10 +67,16 @@ type chunkFileItem struct {
Path string
}

func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) {
func listChunksByRunID(st storage.ObjectStorage, runID int64, basepath string) (map[int64][]*chunkFileItem, error) {
storageDir := fmt.Sprintf("tmp%d", runID)
var chunks []*chunkFileItem
if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error {
// if basepath is settings.Actions.ArtifactStorage.MinioConfig.BasePath, it should trim it
if basepath != "" {
log.Debug("listChunksByRunID, trim basepath: %s, path: %s", basepath, path)
basepath = strings.TrimPrefix(basepath, "/")
path = strings.TrimPrefix(path, basepath+"/")
}
fuxiaohei marked this conversation as resolved.
Show resolved Hide resolved
item := chunkFileItem{Path: path}
if _, err := fmt.Sscanf(path, filepath.Join(storageDir, "%d-%d-%d.chunk"), &item.ArtifactID, &item.Start, &item.End); err != nil {
return fmt.Errorf("parse content range error: %v", err)
Expand All @@ -94,8 +103,11 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int
if err != nil {
return err
}
// if use minio, it should trim basepath when iterate objects for chunks
// if use local, basepath is empty
basepath := setting.Actions.ArtifactStorage.MinioConfig.BasePath
// read all uploading chunks from storage
chunksMap, err := listChunksByRunID(st, runID)
chunksMap, err := listChunksByRunID(st, runID, basepath)
if err != nil {
return err
}
Expand Down
Loading