Skip to content

Commit

Permalink
Open parts files one at a time
Browse files Browse the repository at this point in the history
When completing a file-based upload, open the parts files one at a time
and write them to the upload, closing each file before opening the next
one.

This is preferrable to opening them all at once and closing all files at
the end, because it consumes less file descriptors.

Updates #10660
  • Loading branch information
zmb3 committed Mar 4, 2022
1 parent f5029f8 commit ce5e638
Showing 1 changed file with 15 additions and 19 deletions.
34 changes: 15 additions & 19 deletions lib/events/filesessions/filestream.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,30 +124,26 @@ func (h *Handler) CompleteUpload(ctx context.Context, upload events.StreamUpload
}
}()

files := make([]*os.File, 0, len(parts))
readers := make([]io.Reader, 0, len(parts))

defer func() {
for i := 0; i < len(files); i++ {
if err := files[i].Close(); err != nil {
h.WithError(err).Errorf("Failed to close file %q.", files[i].Name())
}
writePartToFile := func(path string) error {
file, err := os.Open(path)
if err != nil {
return err
}
}()
defer func() {
if err := file.Close(); err != nil {
h.WithError(err).Errorf("failed to close file %q", path)
}
}()

_, err = io.Copy(f, file)
return err
}

for _, part := range parts {
partPath := h.partPath(upload, part.Number)
file, err := os.Open(partPath)
if err != nil {
return trace.ConvertSystemError(err)
if err := writePartToFile(partPath); err != nil {
return trace.Wrap(err)
}
files = append(files, file)
readers = append(readers, file)
}

_, err = io.Copy(f, io.MultiReader(readers...))
if err != nil {
return trace.Wrap(err)
}

err = h.Config.OnBeforeComplete(ctx, upload)
Expand Down

0 comments on commit ce5e638

Please sign in to comment.