Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split buffer and file download options into their own types #18896

Merged
merged 2 commits into from
Aug 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 14 additions & 9 deletions sdk/storage/azblob/blob/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,7 @@ func (b *Client) GetSASToken(permissions SASPermissions, start time.Time, expiry
// Concurrent Download Functions -----------------------------------------------------------------------------------------

// download downloads an Azure blob to a WriterAt in parallel.
func (b *Client) download(ctx context.Context, writer io.WriterAt, o *downloadOptions) (int64, error) {
if o == nil {
o = &downloadOptions{}
}
func (b *Client) download(ctx context.Context, writer io.WriterAt, o downloadOptions) (int64, error) {
if o.BlockSize == 0 {
o.BlockSize = DefaultDownloadBlockSize
}
Expand Down Expand Up @@ -372,24 +369,32 @@ func (b *Client) DownloadStream(ctx context.Context, o *DownloadStreamOptions) (

// DownloadBuffer downloads an Azure blob to a buffer with parallel.
func (b *Client) DownloadBuffer(ctx context.Context, buffer []byte, o *DownloadBufferOptions) (int64, error) {
return b.download(ctx, shared.NewBytesWriter(buffer), o)
if o == nil {
o = &DownloadBufferOptions{}
}
return b.download(ctx, shared.NewBytesWriter(buffer), (downloadOptions)(*o))
}

// DownloadFile downloads an Azure blob to a local file.
// The file would be truncated if the size doesn't match.
func (b *Client) DownloadFile(ctx context.Context, file *os.File, o *DownloadFileOptions) (int64, error) {
if o == nil {
o = &DownloadFileOptions{}
}
do := (*downloadOptions)(o)

// 1. Calculate the size of the destination file
var size int64

count := o.Count
count := do.Count
if count == CountToEnd {
// Try to get Azure blob's size
getBlobPropertiesOptions := o.getBlobPropertiesOptions()
getBlobPropertiesOptions := do.getBlobPropertiesOptions()
props, err := b.GetProperties(ctx, getBlobPropertiesOptions)
jhendrixMSFT marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return 0, err
}
size = *props.ContentLength - o.Offset
size = *props.ContentLength - do.Offset
} else {
size = count
}
Expand All @@ -406,7 +411,7 @@ func (b *Client) DownloadFile(ctx context.Context, file *os.File, o *DownloadFil
}

if size > 0 {
return b.download(ctx, file, o)
return b.download(ctx, file, *do)
} else { // if the blob's size is 0, there is no need in downloading it
return 0, nil
}
Expand Down
66 changes: 61 additions & 5 deletions sdk/storage/azblob/blob/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (o *DownloadStreamOptions) format() (*generated.BlobClientDownloadOptions,

// ---------------------------------------------------------------------------------------------------------------------

// downloadOptions identifies options used by the DownloadBuffer and DownloadFile functions.
// downloadOptions contains common options used by the DownloadBuffer and DownloadFile functions.
type downloadOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64
Expand Down Expand Up @@ -138,13 +138,19 @@ type downloadOptions struct {
}

func (o *downloadOptions) getBlobPropertiesOptions() *GetPropertiesOptions {
if o == nil {
return nil
}
return &GetPropertiesOptions{
AccessConditions: o.AccessConditions,
CpkInfo: o.CpkInfo,
}
}

func (o *downloadOptions) getDownloadBlobOptions(offSet, count int64, rangeGetContentMD5 *bool) *DownloadStreamOptions {
if o == nil {
return nil
}
return &DownloadStreamOptions{
AccessConditions: o.AccessConditions,
CpkInfo: o.CpkInfo,
Expand All @@ -155,11 +161,61 @@ func (o *downloadOptions) getDownloadBlobOptions(offSet, count int64, rangeGetCo
}
}

// DownloadBufferOptions identifies options used by the DownloadBuffer and DownloadFile functions.
type DownloadBufferOptions = downloadOptions
// DownloadBufferOptions contains the optional parameters for the DownloadBuffer method.
type DownloadBufferOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64

// Offset is the byte offset within the blob to start the download. The default value is zero.
Offset int64

// BlockSize specifies the block size to use for each parallel download; the default size is DefaultDownloadBlockSize.
BlockSize int64

// Progress is a function that is invoked periodically as bytes are received.
Progress func(bytesTransferred int64)

// BlobAccessConditions indicates the access conditions used when making HTTP GET requests against the blob.
AccessConditions *AccessConditions

// ClientProvidedKeyOptions indicates the client provided key by name and/or by value to encrypt/decrypt data.
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
}

// DownloadFileOptions contains the optional parameters for the DownloadFile method.
type DownloadFileOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64

// Offset is the byte offset within the blob to start the download. The default value is zero.
Offset int64

// BlockSize specifies the block size to use for each parallel download; the default size is DefaultDownloadBlockSize.
BlockSize int64

// Progress is a function that is invoked periodically as bytes are received.
Progress func(bytesTransferred int64)

// DownloadFileOptions identifies options used by the DownloadBuffer and DownloadFile functions.
type DownloadFileOptions = downloadOptions
// BlobAccessConditions indicates the access conditions used when making HTTP GET requests against the blob.
AccessConditions *AccessConditions

// ClientProvidedKeyOptions indicates the client provided key by name and/or by value to encrypt/decrypt data.
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
}

// ---------------------------------------------------------------------------------------------------------------------

Expand Down