From ed498a51978901f347293da7a065d369b0eb7e24 Mon Sep 17 00:00:00 2001 From: Andreas Skorczyk Date: Fri, 30 Jul 2021 16:47:23 +0200 Subject: [PATCH] s3ng: Provide objectSize when uploading According to the documentation for `PutObject`, setting objectSize to -1 causes a multipart Put operation until the input stream reaches EOF, which can result in high memory usage. Something similar has already been reported in https://github.com/minio/minio-go/issues/1496 This PR changes the behavior so it tries to determine the file size before uploading it. Should the given io.Reader not be a file, it falls back to using `-1`. In my testing this change has reduced the memory usage when uploading files to OICS quite a bit: From over 1GB for a single user, down to <300MB. --- changelog/unreleased/provide-size-for-s3-upload.md | 7 +++++++ pkg/storage/fs/s3ng/blobstore/blobstore.go | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/provide-size-for-s3-upload.md diff --git a/changelog/unreleased/provide-size-for-s3-upload.md b/changelog/unreleased/provide-size-for-s3-upload.md new file mode 100644 index 0000000000..f2c92943a9 --- /dev/null +++ b/changelog/unreleased/provide-size-for-s3-upload.md @@ -0,0 +1,7 @@ +Enhancement: Reduce memory usage when uploading with S3ng storage + +The memory usage could be high when uploading files using the S3ng storage. +By providing the actual file size when triggering `PutObject`, +the overall memory usage is reduced. + +https://github.com/cs3org/reva/pull/1940 diff --git a/pkg/storage/fs/s3ng/blobstore/blobstore.go b/pkg/storage/fs/s3ng/blobstore/blobstore.go index 4f44268a4e..0599d48f35 100644 --- a/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ b/pkg/storage/fs/s3ng/blobstore/blobstore.go @@ -22,6 +22,7 @@ import ( "context" "io" "net/url" + "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -60,7 +61,16 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, err // Upload stores some data in the blobstore under the given key func (bs *Blobstore) Upload(key string, reader io.Reader) error { - _, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + size := int64(-1) + if file, ok := reader.(*os.File); ok { + info, err := file.Stat() + if err != nil { + return errors.Wrapf(err, "could not determine file size for object '%s'", key) + } + size = info.Size() + } + + _, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, size, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", key, bs.bucket)