diff --git a/pkg/cloud/amazon/s3_storage.go b/pkg/cloud/amazon/s3_storage.go index 6f07f1fb4b8d..696cb4f82f70 100644 --- a/pkg/cloud/amazon/s3_storage.go +++ b/pkg/cloud/amazon/s3_storage.go @@ -352,7 +352,9 @@ func newClient( sess.Config.Region = aws.String(region) c := s3.New(sess) - u := s3manager.NewUploader(sess) + u := s3manager.NewUploader(sess, func(uploader *s3manager.Uploader) { + uploader.PartSize = cloud.WriteChunkSize.Get(&settings.SV) + }) return s3Client{client: c, uploader: u}, region, nil } diff --git a/pkg/cloud/azure/azure_storage.go b/pkg/cloud/azure/azure_storage.go index 310124458493..bc659d8ba134 100644 --- a/pkg/cloud/azure/azure_storage.go +++ b/pkg/cloud/azure/azure_storage.go @@ -125,7 +125,7 @@ func (s *azureStorage) Writer(ctx context.Context, basename string) (io.WriteClo defer sp.Finish() _, err := azblob.UploadStreamToBlockBlob( ctx, r, blob, azblob.UploadStreamToBlockBlobOptions{ - BufferSize: 4 << 20, + BufferSize: int(cloud.WriteChunkSize.Get(&s.settings.SV)), }, ) return err diff --git a/pkg/cloud/cloud_io.go b/pkg/cloud/cloud_io.go index e452d89d85bc..191d58c78511 100644 --- a/pkg/cloud/cloud_io.go +++ b/pkg/cloud/cloud_io.go @@ -49,6 +49,15 @@ var httpCustomCA = settings.RegisterStringSetting( "", ).WithPublic() +// WriteChunkSize is used to control the size of each chunk that is buffered and +// uploaded by the cloud storage client. +var WriteChunkSize = settings.RegisterByteSizeSetting( + settings.TenantWritable, + "cloudstorage.write_chunk.size", + "controls the size of each file chunk uploaded by the cloud storage client", + 8<<20, +) + // HTTPRetryOptions defines the tunable settings which control the retry of HTTP // operations. var HTTPRetryOptions = retry.Options{ diff --git a/pkg/cloud/gcp/gcs_storage.go b/pkg/cloud/gcp/gcs_storage.go index 2bd8bf7233ea..3ce1d6c9c83f 100644 --- a/pkg/cloud/gcp/gcs_storage.go +++ b/pkg/cloud/gcp/gcs_storage.go @@ -162,6 +162,7 @@ func (g *gcsStorage) Writer(ctx context.Context, basename string) (io.WriteClose path.Join(g.prefix, basename))}) w := g.bucket.Object(path.Join(g.prefix, basename)).NewWriter(ctx) + w.ChunkSize = int(cloud.WriteChunkSize.Get(&g.settings.SV)) if !gcsChunkingEnabled.Get(&g.settings.SV) { w.ChunkSize = 0 }