Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Azure Blob Storage Client Library for Go #3

Merged
merged 48 commits into from
Nov 26, 2014
Merged
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
6c8f330
initial client model
ahmetb Nov 7, 2014
5dc6b83
import authorization funcs from packer
ahmetb Nov 18, 2014
f39be88
add more calls before refactoring
ahmetb Nov 18, 2014
b4b95be
unit tests for some helpers
ahmetb Nov 18, 2014
b7ee921
refactor buildCanonicalizedString
ahmetb Nov 18, 2014
e4703f7
refactor authorization header generation
ahmetb Nov 18, 2014
a1a9e70
types for blob type, container access level
ahmetb Nov 18, 2014
d6e4dcf
list containers with deserialization
ahmetb Nov 18, 2014
1e4bee7
list containers parameters & pagination tests
ahmetb Nov 19, 2014
d7ed9f7
refactor client.exec return type
ahmetb Nov 19, 2014
0a877da
put blob (small) validation
ahmetb Nov 19, 2014
c9cf987
get rid of ContainerList type
ahmetb Nov 19, 2014
9381c75
ListBlobs with tests
ahmetb Nov 19, 2014
32b4cdc
test cleanup
ahmetb Nov 19, 2014
97e9ac1
change test for container access mode coverage
ahmetb Nov 19, 2014
523e538
store key as decoded b64, bubble up error early
ahmetb Nov 19, 2014
2ceb06e
refactor PutBlob into PutBlockBlob
ahmetb Nov 20, 2014
102fe60
put multi-block blob
ahmetb Nov 20, 2014
cc01e39
ContainerExists, fix bug in BlobExists, use HEAD instead of GET
ahmetb Nov 20, 2014
6dc2433
errors cosmetics fix
ahmetb Nov 20, 2014
0def2c4
fix return types of CreateContainer/DeleteContainer
ahmetb Nov 20, 2014
f7abd99
fix return type of DeleteBlob
ahmetb Nov 20, 2014
1040339
add DeleteBlobIfExists
ahmetb Nov 20, 2014
574051a
add DeleteContainerIfExists
ahmetb Nov 20, 2014
63fad88
add CreateContainerIfNotExists
ahmetb Nov 20, 2014
7aac886
return io.ReadCloser from exec
ahmetb Nov 20, 2014
b2d81da
return io.ReadCloser from GetBlob
ahmetb Nov 20, 2014
0833a6e
get blob properties
ahmetb Nov 20, 2014
34159aa
cosmetics cleanup
ahmetb Nov 20, 2014
4f76552
copy blob
ahmetb Nov 20, 2014
8439bef
status code error fixes
ahmetb Nov 20, 2014
a17a775
add GetBlobSASURI
ahmetb Nov 20, 2014
964910b
fix content-length for single block blobs, add regression test
ahmetb Nov 20, 2014
6b5f9ef
return StorageServiceError from responses
ahmetb Nov 24, 2014
cfc2ddd
rearrange definitions
ahmetb Nov 24, 2014
f1597ec
comment fix
ahmetb Nov 24, 2014
3bf1915
add GetBlockList, expose PutBlock and PutBlockList
ahmetb Nov 25, 2014
23c9921
relocate TestReturnedServiceError
ahmetb Nov 25, 2014
fa6c872
plumb request id header to StorageServiceError
ahmetb Nov 25, 2014
2845a47
fix error message
ahmetb Nov 25, 2014
5f7059a
block list verification for put single block blob
ahmetb Nov 25, 2014
cee0d7e
add bool flags to indicate operation effect for *Exists$ methods
ahmetb Nov 25, 2014
9285751
implement GetBlockRange and tests
ahmetb Nov 25, 2014
a52b368
make ContentLength type uint64
ahmetb Nov 25, 2014
a5aaf7f
export BlockStatus strings
ahmetb Nov 25, 2014
2236a3d
fix compilation bug
ahmetb Nov 25, 2014
aee6a4a
add PutBlockWithSize for streams with size known in advance
ahmetb Nov 26, 2014
ffcabd1
fix max block size
ahmetb Nov 26, 2014
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add more calls before refactoring
  • Loading branch information
ahmetb committed Nov 26, 2014
commit f39be88b61b60e68f7989fa7c683a7785efc0e3e
172 changes: 172 additions & 0 deletions clients/storage/blob.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package storage

import (
"bytes"
"fmt"
"net/http"
"net/url"
"time"
@@ -35,7 +37,14 @@ type ContainerListResponse struct {
Containers []Container
}

const (
BlobTypeBlock = "BlockBlob"
BlobTypePage = "PageBlob"
)

func (b BlobStorageClient) ListContainers() (*http.Response, error) {
// TODO (ahmetb) pagination

verb := "GET"
uri := b.client.getEndpoint(blobServiceName, "", url.Values{"comp": {"list"}})

@@ -74,3 +83,166 @@ func (b BlobStorageClient) ListContainers() (*http.Response, error) {

return resp, nil
}

func (b BlobStorageClient) GetContainer(name string) (*http.Response, error) {
verb := "GET"
uri := b.client.getEndpoint(blobServiceName, name, url.Values{"restype": {"container"}})

headers := b.client.getStandardHeaders()
canonicalizedHeaders := b.client.buildCanonicalizedHeader(headers)
canonicalizedResource, err := b.client.buildCanonicalizedResource(uri)

if err != nil {
return nil, err
}

contentEncoding := ""
contentLanguage := ""
contentLength := "" // difference here!
contentMD5 := ""
contentType := ""
date := ""
ifModifiedSince := ""
ifMatch := ""
ifNoneMatch := ""
ifUnmodifiedSince := ""
Range := ""

canonicalizedString := b.client.buildCanonicalizedString(verb, contentEncoding, contentLanguage, contentLength, contentMD5, contentType,
date, ifModifiedSince, ifMatch, ifNoneMatch, ifUnmodifiedSince, Range, canonicalizedHeaders, canonicalizedResource)
authHeader, err := b.client.createAuthorizationHeader(canonicalizedString)
if err != nil {
return nil, err
}

headers["Authorization"] = authHeader
resp, err := b.client.exec(verb, uri, headers, nil)
if err != nil {
return nil, err
}

return resp, nil
}

func (b BlobStorageClient) CreateContainer(name string) (*http.Response, error) {
verb := "PUT"
uri := b.client.getEndpoint(blobServiceName, name, url.Values{"restype": {"container"}})

headers := b.client.getStandardHeaders()
canonicalizedHeaders := b.client.buildCanonicalizedHeader(headers)
canonicalizedResource, err := b.client.buildCanonicalizedResource(uri)

if err != nil {
return nil, err
}

contentEncoding := ""
contentLanguage := ""
contentLength := "0" // difference here!
contentMD5 := ""
contentType := ""
date := ""
ifModifiedSince := ""
ifMatch := ""
ifNoneMatch := ""
ifUnmodifiedSince := ""
Range := ""

canonicalizedString := b.client.buildCanonicalizedString(verb, contentEncoding, contentLanguage, contentLength, contentMD5, contentType,
date, ifModifiedSince, ifMatch, ifNoneMatch, ifUnmodifiedSince, Range, canonicalizedHeaders, canonicalizedResource)
authHeader, err := b.client.createAuthorizationHeader(canonicalizedString)
if err != nil {
return nil, err
}

headers["Authorization"] = authHeader
resp, err := b.client.exec(verb, uri, headers, nil)
if err != nil {
return nil, err
}

return resp, nil
}

func (b BlobStorageClient) DeleteContainer(name string) (*http.Response, error) {
verb := "DELETE"
uri := b.client.getEndpoint(blobServiceName, name, url.Values{"restype": {"container"}})

headers := b.client.getStandardHeaders()
canonicalizedHeaders := b.client.buildCanonicalizedHeader(headers)
canonicalizedResource, err := b.client.buildCanonicalizedResource(uri)

if err != nil {
return nil, err
}

contentEncoding := ""
contentLanguage := ""
contentLength := ""
contentMD5 := ""
contentType := ""
date := ""
ifModifiedSince := ""
ifMatch := ""
ifNoneMatch := ""
ifUnmodifiedSince := ""
Range := ""

canonicalizedString := b.client.buildCanonicalizedString(verb, contentEncoding, contentLanguage, contentLength, contentMD5, contentType,
date, ifModifiedSince, ifMatch, ifNoneMatch, ifUnmodifiedSince, Range, canonicalizedHeaders, canonicalizedResource)
authHeader, err := b.client.createAuthorizationHeader(canonicalizedString)
if err != nil {
return nil, err
}

headers["Authorization"] = authHeader
resp, err := b.client.exec(verb, uri, headers, nil)
if err != nil {
return nil, err
}

return resp, nil
}

func (b BlobStorageClient) PutBlob(name, container, blobType string, blob []byte) (*http.Response, error) {
verb := "PUT"
path := fmt.Sprintf("%s/%s", name, container)
uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
blobSize := len(blob)

headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = blobType
canonicalizedHeaders := b.client.buildCanonicalizedHeader(headers)
canonicalizedResource, err := b.client.buildCanonicalizedResource(uri)

if err != nil {
return nil, err
}

contentEncoding := ""
contentLanguage := ""
contentLength := fmt.Sprintf("%v", blobSize)
contentMD5 := ""
contentType := ""
date := ""
ifModifiedSince := ""
ifMatch := ""
ifNoneMatch := ""
ifUnmodifiedSince := ""
Range := ""

canonicalizedString := b.client.buildCanonicalizedString(verb, contentEncoding, contentLanguage, contentLength, contentMD5, contentType,
date, ifModifiedSince, ifMatch, ifNoneMatch, ifUnmodifiedSince, Range, canonicalizedHeaders, canonicalizedResource)
authHeader, err := b.client.createAuthorizationHeader(canonicalizedString)
if err != nil {
return nil, err
}

headers["Authorization"] = authHeader
resp, err := b.client.exec(verb, uri, headers, bytes.NewReader(blob))
if err != nil {
return nil, err
}

return resp, nil
}
79 changes: 74 additions & 5 deletions clients/storage/blob_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package storage

import (
"crypto/rand"
"errors"
"io/ioutil"
"os"
@@ -13,11 +14,9 @@ func TestListContainers(t *testing.T) {
t.Error(err)
}

resp, err := cli.GetBlobService().ListContainers()
resp, err := cli.ListContainers()
if err != nil {
t.Error(err)
} else if resp == nil {
t.Error("Got nil response")
}

defer resp.Body.Close()
@@ -28,7 +27,63 @@ func TestListContainers(t *testing.T) {
t.Log(string(bytes))
}

func getClient() (*StorageClient, error) {
func TestCreateGetDeleteContainer(t *testing.T) {
cnt := randString(10)

cli, err := getClient()
if err != nil {
t.Error(err)
}

resp1, err := cli.CreateContainer(cnt)
if err != nil {
t.Error(err)
}
t.Logf("Create: %v", resp1.Status)

resp2, err := cli.GetContainer(cnt)
if err != nil {
t.Error(err)
}
t.Logf("Get: %v", resp2.Status)

resp3, err := cli.DeleteContainer(cnt)
if err != nil {
t.Error(err)
}
t.Logf("Delete: %v", resp3.Status)
}

func TestPutBlockBlob(t *testing.T) {
cnt := randString(5)
blob := randString(10)
body := randString(1024 * 4)

cli, err := getClient()
if err != nil {
t.Error(err)
}

resp, err := cli.CreateContainer(cnt)
if err != nil {
t.Error(err)
}
t.Logf("Create container: %v", resp.Status)

resp, err = cli.PutBlob(cnt, blob, BlobTypeBlock, []byte(body))
if err != nil {
t.Error(err)
}
t.Logf("Put blob: %v", resp.Status)

resp, err = cli.DeleteContainer(cnt)
if err != nil {
t.Error(err)
}
t.Logf("Delete container: %v", resp.Status)
}

func getClient() (*BlobStorageClient, error) {
name := os.Getenv("ACCOUNT_NAME")
if name == "" {
return nil, errors.New("ACCOUNT_NAME not set")
@@ -37,5 +92,19 @@ func getClient() (*StorageClient, error) {
if key == "" {
return nil, errors.New("ACCOUNT_KEY not set")
}
return NewBasicClient(name, key)
cli, err := NewBasicClient(name, key)
if err != nil {
return nil, err
}
return cli.GetBlobService(), nil
}

func randString(n int) string {
const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
rand.Read(bytes)
for i, b := range bytes {
bytes[i] = alphanum[b%byte(len(alphanum))]
}
return string(bytes)
}
1 change: 1 addition & 0 deletions clients/storage/client.go
Original file line number Diff line number Diff line change
@@ -206,6 +206,7 @@ func (c StorageClient) buildCanonicalizedString(verb, contentEncoding, contentLa
}

func (c StorageClient) exec(verb, url string, headers map[string]string, body io.Reader) (resp *http.Response, err error) {
fmt.Println(url) // TODO (ahmetalpbalkan) remove
// TODO (ahmetalpbalkan) write test case for imported code
req, err := http.NewRequest(verb, url, body)