Skip to content

Commit

Permalink
Split buffer and file download options into their own types (#18896)
Browse files Browse the repository at this point in the history
* Split buffer and file download options into their own types

This allows the options for each API to independently evolve over time.
Fixed potential panic in some helper methods.

* type-o
  • Loading branch information
jhendrixMSFT authored Aug 18, 2022
1 parent f320fb0 commit 03952f8
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 14 deletions.
23 changes: 14 additions & 9 deletions sdk/storage/azblob/blob/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,7 @@ func (b *Client) GetSASToken(permissions SASPermissions, start time.Time, expiry
// Concurrent Download Functions -----------------------------------------------------------------------------------------

// download downloads an Azure blob to a WriterAt in parallel.
func (b *Client) download(ctx context.Context, writer io.WriterAt, o *downloadOptions) (int64, error) {
if o == nil {
o = &downloadOptions{}
}
func (b *Client) download(ctx context.Context, writer io.WriterAt, o downloadOptions) (int64, error) {
if o.BlockSize == 0 {
o.BlockSize = DefaultDownloadBlockSize
}
Expand Down Expand Up @@ -372,24 +369,32 @@ func (b *Client) DownloadStream(ctx context.Context, o *DownloadStreamOptions) (

// DownloadBuffer downloads an Azure blob to a buffer with parallel.
func (b *Client) DownloadBuffer(ctx context.Context, buffer []byte, o *DownloadBufferOptions) (int64, error) {
return b.download(ctx, shared.NewBytesWriter(buffer), o)
if o == nil {
o = &DownloadBufferOptions{}
}
return b.download(ctx, shared.NewBytesWriter(buffer), (downloadOptions)(*o))
}

// DownloadFile downloads an Azure blob to a local file.
// The file would be truncated if the size doesn't match.
func (b *Client) DownloadFile(ctx context.Context, file *os.File, o *DownloadFileOptions) (int64, error) {
if o == nil {
o = &DownloadFileOptions{}
}
do := (*downloadOptions)(o)

// 1. Calculate the size of the destination file
var size int64

count := o.Count
count := do.Count
if count == CountToEnd {
// Try to get Azure blob's size
getBlobPropertiesOptions := o.getBlobPropertiesOptions()
getBlobPropertiesOptions := do.getBlobPropertiesOptions()
props, err := b.GetProperties(ctx, getBlobPropertiesOptions)
if err != nil {
return 0, err
}
size = *props.ContentLength - o.Offset
size = *props.ContentLength - do.Offset
} else {
size = count
}
Expand All @@ -406,7 +411,7 @@ func (b *Client) DownloadFile(ctx context.Context, file *os.File, o *DownloadFil
}

if size > 0 {
return b.download(ctx, file, o)
return b.download(ctx, file, *do)
} else { // if the blob's size is 0, there is no need in downloading it
return 0, nil
}
Expand Down
66 changes: 61 additions & 5 deletions sdk/storage/azblob/blob/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (o *DownloadStreamOptions) format() (*generated.BlobClientDownloadOptions,

// ---------------------------------------------------------------------------------------------------------------------

// downloadOptions identifies options used by the DownloadBuffer and DownloadFile functions.
// downloadOptions contains common options used by the DownloadBuffer and DownloadFile functions.
type downloadOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64
Expand Down Expand Up @@ -138,13 +138,19 @@ type downloadOptions struct {
}

func (o *downloadOptions) getBlobPropertiesOptions() *GetPropertiesOptions {
if o == nil {
return nil
}
return &GetPropertiesOptions{
AccessConditions: o.AccessConditions,
CpkInfo: o.CpkInfo,
}
}

func (o *downloadOptions) getDownloadBlobOptions(offSet, count int64, rangeGetContentMD5 *bool) *DownloadStreamOptions {
if o == nil {
return nil
}
return &DownloadStreamOptions{
AccessConditions: o.AccessConditions,
CpkInfo: o.CpkInfo,
Expand All @@ -155,11 +161,61 @@ func (o *downloadOptions) getDownloadBlobOptions(offSet, count int64, rangeGetCo
}
}

// DownloadBufferOptions identifies options used by the DownloadBuffer and DownloadFile functions.
type DownloadBufferOptions = downloadOptions
// DownloadBufferOptions contains the optional parameters for the DownloadBuffer method.
type DownloadBufferOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64

// Offset is the byte offset within the blob to start the download. The default value is zero.
Offset int64

// BlockSize specifies the block size to use for each parallel download; the default size is DefaultDownloadBlockSize.
BlockSize int64

// Progress is a function that is invoked periodically as bytes are received.
Progress func(bytesTransferred int64)

// BlobAccessConditions indicates the access conditions used when making HTTP GET requests against the blob.
AccessConditions *AccessConditions

// ClientProvidedKeyOptions indicates the client provided key by name and/or by value to encrypt/decrypt data.
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
}

// DownloadFileOptions contains the optional parameters for the DownloadFile method.
type DownloadFileOptions struct {
// Count is the number of bytes to download. Specify 0 to download the entire blob (this is the default).
Count int64

// Offset is the byte offset within the blob to start the download. The default value is zero.
Offset int64

// BlockSize specifies the block size to use for each parallel download; the default size is DefaultDownloadBlockSize.
BlockSize int64

// Progress is a function that is invoked periodically as bytes are received.
Progress func(bytesTransferred int64)

// DownloadFileOptions identifies options used by the DownloadBuffer and DownloadFile functions.
type DownloadFileOptions = downloadOptions
// BlobAccessConditions indicates the access conditions used when making HTTP GET requests against the blob.
AccessConditions *AccessConditions

// ClientProvidedKeyOptions indicates the client provided key by name and/or by value to encrypt/decrypt data.
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
}

// ---------------------------------------------------------------------------------------------------------------------

Expand Down

0 comments on commit 03952f8

Please sign in to comment.