Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel download #7429

Merged
merged 1 commit into from
Aug 30, 2019
Merged

Parallel download #7429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
568 changes: 568 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs
View file
Open in desktop

Large diffs are not rendered by default.

90 changes: 90 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public void Ctor_ConnectionString()
Assert.AreEqual("accountName", builder.AccountName);
}

#region Sequential Download

[Test]
public async Task DownloadAsync()
{
Expand Down Expand Up @@ -307,6 +309,94 @@ async Task Verify(Response<BlobDownloadInfo> response)
}
}
}
#endregion Sequential Download

#region Parallel Download

private async Task ParallelDownloadFileAndVerify(
long size,
long singleBlockThreshold,
ParallelTransferOptions parallelTransferOptions)
{
var data = this.GetRandomBuffer(size);
var path = Path.GetTempFileName();

try
{
using (this.GetNewContainer(out var container))
{
var name = this.GetNewBlobName();
var blob = this.InstrumentClient(container.GetBlobClient(name));

using (var stream = new MemoryStream(data))
{
await blob.UploadAsync(stream);
}

var destination = new FileInfo(path);

await blob.StagedDownloadAsync(
destination,
singleBlockThreshold: singleBlockThreshold,
parallelTransferOptions: parallelTransferOptions
);

using (var resultStream = destination.OpenRead())
{
TestHelper.AssertSequenceEqual(data, resultStream.AsBytes());
}
}
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}

[Test]
[TestCase(512)]
[TestCase(1 * Constants.KB)]
[TestCase(2 * Constants.KB)]
[TestCase(4 * Constants.KB)]
[TestCase(10 * Constants.KB)]
[TestCase(20 * Constants.KB)]
[TestCase(30 * Constants.KB)]
[TestCase(50 * Constants.KB)]
[TestCase(501 * Constants.KB)]
public async Task DownloadFileAsync_Parallel_SmallBlobs(long size) =>
// Use a 1KB threshold so we get a lot of individual blocks
await this.ParallelDownloadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });

[Test]
[Category("Live")]
[TestCase(33 * Constants.MB, 1)]
[TestCase(33 * Constants.MB, 4)]
[TestCase(33 * Constants.MB, 8)]
[TestCase(33 * Constants.MB, 16)]
[TestCase(33 * Constants.MB, null)]
[TestCase(257 * Constants.MB, 1)]
[TestCase(257 * Constants.MB, 4)]
[TestCase(257 * Constants.MB, 8)]
[TestCase(257 * Constants.MB, 16)]
[TestCase(257 * Constants.MB, null)]
[TestCase(1 * Constants.GB, 1)]
[TestCase(1 * Constants.GB, 4)]
[TestCase(1 * Constants.GB, 8)]
[TestCase(1 * Constants.GB, 16)]
[TestCase(1 * Constants.GB, null)]
public async Task DownloadFileAsync_Parallel_LargeBlobs(long size, int? maximumThreadCount)
{
// TODO: #6781 We don't want to add 1GB of random data in the recordings
if (this.Mode == RecordedTestMode.Live)
{
await this.ParallelDownloadFileAndVerify(size, 16 * Constants.MB, new ParallelTransferOptions { MaximumThreadCount = maximumThreadCount });
}
}

#endregion Parallel Download

[Test]
public async Task StartCopyFromUriAsync()
Expand Down
8 changes: 6 additions & 2 deletions sdk/storage/Azure.Storage.Blobs/tests/BlobClientTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ public void Ctor_ConnectionString()
Assert.AreEqual("accountName", builder.AccountName);
}

#region Upload

[Test]
public async Task UploadAsync_Stream()
{
Expand Down Expand Up @@ -446,7 +448,7 @@ await blob.StagedUploadAsync(
[TestCase(501 * Constants.KB)]
public async Task UploadStreamAsync_SmallBlobs(long size) =>
// Use a 1KB threshold so we get a lot of individual blocks
await this.UploadStreamAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumBlockLength = Constants.KB });
await this.UploadStreamAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });

[Test]
[TestCase(512)]
Expand All @@ -460,7 +462,7 @@ public async Task UploadStreamAsync_SmallBlobs(long size) =>
[TestCase(501 * Constants.KB)]
public async Task UploadFileAsync_SmallBlobs(long size) =>
// Use a 1KB threshold so we get a lot of individual blocks
await this.UploadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumBlockLength = Constants.KB });
await this.UploadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });

[Test]
[TestCase(33 * Constants.MB, 1)]
Expand Down Expand Up @@ -512,5 +514,7 @@ public async Task UploadFileAsync_LargeBlobs(long size, int? maximumThreadCount)
await this.UploadFileAndVerify(size, 16 * Constants.MB, new ParallelTransferOptions { MaximumThreadCount = maximumThreadCount });
}
}

#endregion Upload
}
}
4 changes: 4 additions & 0 deletions ...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...cords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
4 changes: 4 additions & 0 deletions ...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null)Async.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"Entries": [],
"Variables": {}
}
Loading