From 0bfe6be7a44e030413a834ca229b2ae67a430f9d Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Thu, 6 Oct 2016 22:20:16 +1100 Subject: [PATCH 01/31] beginning of blob lease functionality --- storage/blob.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/storage/blob.go b/storage/blob.go index 4207cfec6e7a..fde87e66b620 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -256,6 +256,22 @@ const ( blobCopyStatusFailed = "failed" ) +// lease constants. +const ( + leaseHeaderPrefix = "x-ms-lease-" + leaseID = "x-ms-lease-id" + leaseAction = "x-ms-lease-action" + leaseBreakPeriod = "x-ms-lease-break-period" + leaseDuration = "x-ms-lease-duration" + leaseProposedID = "x-ms-proposed-lease-id" + + acquireLease = "acquire" + renewLease = "renew" + changeLease = "change" + releaseLease = "release" + breakLease = "break" +) + // BlockListType is used to filter out types of blocks in a Get Blocks List call // for a block blob. // @@ -560,6 +576,64 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string, extr return resp, err } +// AcquireLease gets a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (string, error) { + params := url.Values{"comp": {"lease"}} + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + headers := b.client.getStandardHeaders() + + headers[leaseAction] = acquireLease + headers[leaseProposedID] = proposedLeaseID + headers[leaseDuration] = string(leaseTimeInSeconds) + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return "", err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil { + return "", err + } + + for k, v := range resp.headers { + k = strings.ToLower(k) + if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(leaseHeaderPrefix)) { + continue + } + + // we only want the lease ID + if k == leaseID { + return v[0], nil + } + } + + // what should we return in case of HTTP 201 but no lease ID? + // or it just cant happen? (brave words) + + return "", nil +} + +// BreakLease breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) BreakLease(container string, name string, breakPeriodInSeconds int) error { + return nil +} + +// ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) ChangeLease(container string, name string, proposedLeaseID string) (string, error) { + return "", nil +} + +// ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) ReleaseLease(container string, name string) error { + return nil +} + +// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) RenewLease(container string, name string) error { + return nil +} + // GetBlobProperties provides various information about the specified // blob. See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) { From daf1095befd835818d9db560ac79ffbd50e41622 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 14 Oct 2016 18:35:00 +1100 Subject: [PATCH 02/31] breaklease with and without break period working --- storage/blob.go | 52 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index fde87e66b620..6d0e1dbf16c8 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "strconv" @@ -264,6 +265,7 @@ const ( leaseBreakPeriod = "x-ms-lease-break-period" leaseDuration = "x-ms-lease-duration" leaseProposedID = "x-ms-proposed-lease-id" + leaseTime = "x-ms-lease-time" acquireLease = "acquire" renewLease = "renew" @@ -584,7 +586,7 @@ func (b BlobStorageClient) AcquireLease(container string, name string, leaseTime headers[leaseAction] = acquireLease headers[leaseProposedID] = proposedLeaseID - headers[leaseDuration] = string(leaseTimeInSeconds) + headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) resp, err := b.client.exec("PUT", uri, headers, nil) if err != nil { @@ -610,13 +612,55 @@ func (b BlobStorageClient) AcquireLease(container string, name string, leaseTime // what should we return in case of HTTP 201 but no lease ID? // or it just cant happen? (brave words) - return "", nil } // BreakLease breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) BreakLease(container string, name string, breakPeriodInSeconds int) error { - return nil +func (b BlobStorageClient) BreakLease(container string, name string) (int, error) { + headers := b.client.getStandardHeaders() + headers[leaseAction] = breakLease + return b.breakLeaseCommon(container, name, headers) +} + +// BreakLeaseWithBreakPeriod breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name string, breakPeriodInSeconds int) (int, error) { + headers := b.client.getStandardHeaders() + headers[leaseAction] = breakLease + headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds) + return b.breakLeaseCommon(container, name, headers) +} + +// breakLeaseCommon is common code for both version of BreakLease (with and without break period) +func (b BlobStorageClient) breakLeaseCommon(container string, name string, headers map[string]string) (int, error) { + params := url.Values{"comp": {"lease"}} + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return 0, err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted}); err != nil { + return 0, err + } + + for k, v := range resp.headers { + log.Println(k, v) + + k = strings.ToLower(k) + if !strings.HasPrefix(k, strings.ToLower(leaseTime)) { + continue + } + + timeout, err := strconv.Atoi(v[0]) + if err != nil { + return 0, err + } + + return timeout, nil + } + return 0, nil } // ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx From 5161cd306d3dfe13cd21a3c4a3c42ddc86a4f479 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 14 Oct 2016 18:45:18 +1100 Subject: [PATCH 03/31] change lease working --- storage/blob.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index 6d0e1dbf16c8..6e7649e229ee 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -664,7 +664,39 @@ func (b BlobStorageClient) breakLeaseCommon(container string, name string, heade } // ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) ChangeLease(container string, name string, proposedLeaseID string) (string, error) { +func (b BlobStorageClient) ChangeLease(container string, name string, currentLeaseID string, proposedLeaseID string) (string, error) { + params := url.Values{"comp": {"lease"}} + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + headers := b.client.getStandardHeaders() + + headers[leaseAction] = changeLease + headers[leaseID] = currentLeaseID + headers[leaseProposedID] = proposedLeaseID + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return "", err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return "", err + } + + for k, v := range resp.headers { + k = strings.ToLower(k) + if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(leaseHeaderPrefix)) { + continue + } + + // we only want the lease ID + if k == leaseID { + return v[0], nil + } + } + + // what should we return in case of HTTP 201 but no lease ID? + // or it just cant happen? (brave words) return "", nil } From 15e14b16b8535c430ba7c364c4b4d61b00efbfd4 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Sat, 15 Oct 2016 10:10:02 +1100 Subject: [PATCH 04/31] renew lease working --- storage/blob.go | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 6e7649e229ee..48b8fe5d826d 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -701,12 +701,46 @@ func (b BlobStorageClient) ChangeLease(container string, name string, currentLea } // ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) ReleaseLease(container string, name string) error { +func (b BlobStorageClient) ReleaseLease(container string, name string, currentLeaseID string) error { + params := url.Values{"comp": {"lease"}} + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + headers := b.client.getStandardHeaders() + + headers[leaseAction] = releaseLease + headers[leaseID] = currentLeaseID + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return err + } + return nil } // RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) RenewLease(container string, name string) error { +func (b BlobStorageClient) RenewLease(container string, name string, currentLeaseID string) error { + params := url.Values{"comp": {"lease"}} + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + headers := b.client.getStandardHeaders() + + headers[leaseAction] = renewLease + headers[leaseID] = currentLeaseID + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return err + } + return nil } From 4a351370fee7dd2e5145fbc8b0eaa7e55aabc257 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Sat, 15 Oct 2016 23:11:58 +1100 Subject: [PATCH 05/31] refactor common code into leaseCommonPut --- storage/blob.go | 73 ++++++++++++++++++------------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 48b8fe5d826d..3ce545703838 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -578,27 +578,37 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string, extr return resp, err } -// AcquireLease gets a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (string, error) { +// leasePut is common PUT code for the various aquire/release/break etc functions. +func (b BlobStorageClient) leaseCommonPut(container string, name string, headers map[string]string, expectedStatus int) (http.Header, error) { params := url.Values{"comp": {"lease"}} uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - headers := b.client.getStandardHeaders() - - headers[leaseAction] = acquireLease - headers[leaseProposedID] = proposedLeaseID - headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) resp, err := b.client.exec("PUT", uri, headers, nil) if err != nil { - return "", err + return nil, err } defer resp.body.Close() - if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil { + if err := checkRespCode(resp.statusCode, []int{expectedStatus}); err != nil { + return nil, err + } + + return resp.headers, nil +} + +// AcquireLease gets a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (string, error) { + headers := b.client.getStandardHeaders() + headers[leaseAction] = acquireLease + headers[leaseProposedID] = proposedLeaseID + headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) + + respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusCreated) + if err != nil { return "", err } - for k, v := range resp.headers { + for k, v := range respHeaders { k = strings.ToLower(k) if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(leaseHeaderPrefix)) { continue @@ -632,20 +642,13 @@ func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name stri // breakLeaseCommon is common code for both version of BreakLease (with and without break period) func (b BlobStorageClient) breakLeaseCommon(container string, name string, headers map[string]string) (int, error) { - params := url.Values{"comp": {"lease"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - resp, err := b.client.exec("PUT", uri, headers, nil) + respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusAccepted) if err != nil { return 0, err } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted}); err != nil { - return 0, err - } - for k, v := range resp.headers { + for k, v := range respHeaders { log.Println(k, v) k = strings.ToLower(k) @@ -665,25 +668,17 @@ func (b BlobStorageClient) breakLeaseCommon(container string, name string, heade // ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx func (b BlobStorageClient) ChangeLease(container string, name string, currentLeaseID string, proposedLeaseID string) (string, error) { - params := url.Values{"comp": {"lease"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) headers := b.client.getStandardHeaders() - headers[leaseAction] = changeLease headers[leaseID] = currentLeaseID headers[leaseProposedID] = proposedLeaseID - resp, err := b.client.exec("PUT", uri, headers, nil) + respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusOK) if err != nil { return "", err } - defer resp.body.Close() - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return "", err - } - - for k, v := range resp.headers { + for k, v := range respHeaders { k = strings.ToLower(k) if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(leaseHeaderPrefix)) { continue @@ -702,44 +697,28 @@ func (b BlobStorageClient) ChangeLease(container string, name string, currentLea // ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx func (b BlobStorageClient) ReleaseLease(container string, name string, currentLeaseID string) error { - params := url.Values{"comp": {"lease"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) headers := b.client.getStandardHeaders() - headers[leaseAction] = releaseLease headers[leaseID] = currentLeaseID - resp, err := b.client.exec("PUT", uri, headers, nil) + _, err := b.leaseCommonPut(container, name, headers, http.StatusOK) if err != nil { return err } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return err - } return nil } // RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx func (b BlobStorageClient) RenewLease(container string, name string, currentLeaseID string) error { - params := url.Values{"comp": {"lease"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) headers := b.client.getStandardHeaders() - headers[leaseAction] = renewLease headers[leaseID] = currentLeaseID - resp, err := b.client.exec("PUT", uri, headers, nil) + _, err := b.leaseCommonPut(container, name, headers, http.StatusOK) if err != nil { return err } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return err - } return nil } From ab94ed9032349d2de370dbc6de17a00f939c1ca6 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Sun, 16 Oct 2016 14:01:52 +1100 Subject: [PATCH 06/31] all unit tests passing --- storage/blob.go | 2 - storage/blob_test.go | 168 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 3ce545703838..fd8b6f0fc046 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" "strconv" @@ -649,7 +648,6 @@ func (b BlobStorageClient) breakLeaseCommon(container string, name string, heade } for k, v := range respHeaders { - log.Println(k, v) k = strings.ToLower(k) if !strings.HasPrefix(k, strings.ToLower(leaseTime)) { diff --git a/storage/blob_test.go b/storage/blob_test.go index 47e5e114162c..857181955afd 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -678,6 +678,174 @@ func (s *StorageBlobSuite) TestSetBlobProperties(c *chk.C) { c.Check(mPut.ContentLanguage, chk.Equals, props.ContentLanguage) } +func (s *StorageBlobSuite) TestAcquireLeaseWithNoProposedLeaseID(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + _, err := cli.AcquireLease(cnt, blob, 30, "") + c.Assert(err, chk.NotNil) + +} + +func (s *StorageBlobSuite) TestAcquireLeaseWithProposedLeaseID(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + c.Assert(leaseID, chk.Equals, proposedLeaseID) +} + +func (s *StorageBlobSuite) TestAcquireLeaseWithBadProposedLeaseID(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + proposedLeaseID := "badbadbad" + _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.NotNil) +} + +func (s *StorageBlobSuite) TestRenewLeaseSuccessful(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + err = cli.RenewLease(cnt, blob, leaseID) + c.Assert(err, chk.IsNil) +} + +func (s *StorageBlobSuite) TestRenewLeaseAgainstNoCurrentLease(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + badLeaseID := "1f812371-a41d-49e6-b123-f4b542e85144" + err := cli.RenewLease(cnt, blob, badLeaseID) + c.Assert(err, chk.NotNil) +} + +func (s *StorageBlobSuite) TestChangeLeaseSuccessful(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + newProposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fbb" + newLeaseID, err := cli.ChangeLease(cnt, blob, leaseID, newProposedLeaseID) + c.Assert(err, chk.IsNil) + c.Assert(newLeaseID, chk.Equals, newProposedLeaseID) +} + +func (s *StorageBlobSuite) TestChangeLeaseNotSuccessfulbadProposedLeaseID(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + newProposedLeaseID := "1f812371-a41d-49e6-b123-f4b542e" + _, err = cli.ChangeLease(cnt, blob, leaseID, newProposedLeaseID) + c.Assert(err, chk.NotNil) +} + +func (s *StorageBlobSuite) TestReleaseLeaseSuccessful(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + err = cli.ReleaseLease(cnt, blob, leaseID) + c.Assert(err, chk.IsNil) +} + +func (s *StorageBlobSuite) TestReleaseLeaseNotSuccessfulBadLeaseID(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + err = cli.ReleaseLease(cnt, blob, "badleaseid") + c.Assert(err, chk.NotNil) +} + +func (s *StorageBlobSuite) TestBreakLeaseSuccessful(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randName(5) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" + _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) + c.Assert(err, chk.IsNil) + + _, err = cli.BreakLease(cnt, blob) + c.Assert(err, chk.IsNil) +} + func (s *StorageBlobSuite) TestPutEmptyBlockBlob(c *chk.C) { cli := getBlobClient(c) cnt := randContainer() From 080d2ef528e7d68cb4bf5cb55e3a8c189cab98d3 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Sun, 16 Oct 2016 14:05:41 +1100 Subject: [PATCH 07/31] tweak of godoc --- storage/blob.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index fd8b6f0fc046..23d7870d32e1 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -595,7 +595,7 @@ func (b BlobStorageClient) leaseCommonPut(container string, name string, headers return resp.headers, nil } -// AcquireLease gets a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +// AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (string, error) { headers := b.client.getStandardHeaders() headers[leaseAction] = acquireLease @@ -632,6 +632,7 @@ func (b BlobStorageClient) BreakLease(container string, name string) (int, error } // BreakLeaseWithBreakPeriod breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx +// breakPeriodInSeconds is used to determine how long until new lease can be created. func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name string, breakPeriodInSeconds int) (int, error) { headers := b.client.getStandardHeaders() headers[leaseAction] = breakLease From db88e32bb6cf1ed4e989132b3bca2ef9531dd51d Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Tue, 18 Oct 2016 18:20:30 +1100 Subject: [PATCH 08/31] Set container ACL initial code --- storage/blob.go | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index 4207cfec6e7a..419eac23af31 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -284,6 +284,10 @@ const ( ContainerAccessTypeContainer ContainerAccessType = "container" ) +const ( + ContainerAccessHeader string = "x-ms-blob-public-access" +) + // Maximum sizes (per REST API) for various concepts const ( MaxBlobBlockSize = 4 * 1024 * 1024 @@ -399,7 +403,7 @@ func (b BlobStorageClient) createContainer(name string, access ContainerAccessTy headers := b.client.getStandardHeaders() if access != "" { - headers["x-ms-blob-public-access"] = string(access) + headers[ContainerAccessHeader] = string(access) } return b.client.exec(verb, uri, headers, nil) } @@ -421,6 +425,48 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { return false, err } +// SetPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx +func (b BlobStorageClient) SetPermissions(container string, access ContainerAccessType) error { + params := url.Values{"restype": {"container"}, + "comp": {"acl"}} + + return b.setPermissionsCommon(container, access, params) +} + +// SetPermissions sets up container permissions (with timeout in seconds) as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx +func (b BlobStorageClient) SetPermissionsWithTimeout(container string, access ContainerAccessType, timeout int) error { + params := url.Values{"restype": {"container"}, + "comp": {"acl"}, + "timeout": {strconv.Itoa(timeout)}} + + return b.setPermissionsCommon(container, access, params) +} + +// setPermissionsCommon sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx +func (b BlobStorageClient) setPermissionsCommon(container string, access ContainerAccessType, params url.Values) error { + uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) + headers := b.client.getStandardHeaders() + if access != "" { + headers[ContainerAccessHeader] = string(access) + } + + verb := "PUT" + resp, err := b.client.exec(verb, uri, headers, nil) + if err != nil { + return err + } + + if resp != nil { + defer resp.body.Close() + + if resp.statusCode != http.StatusOK { + return errors.New("Unable to set permissions") + } + } + + return nil +} + // DeleteContainer deletes the container with given name on the storage // account. If the container does not exist returns error. // From 473d57d4c2e8c4356d87677b732a22de31e15c4c Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Tue, 18 Oct 2016 18:29:28 +1100 Subject: [PATCH 09/31] change so param is struct and can handle optional properties --- storage/blob.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 419eac23af31..9bf4a129fe19 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -284,6 +284,13 @@ const ( ContainerAccessTypeContainer ContainerAccessType = "container" ) +// ContainerAccessOptions are used when setting ACLs of containers (after creation) +type ContainerAccessOptions struct { + ContainerAccess ContainerAccessType + Timeout int + LeaseID string +} + const ( ContainerAccessHeader string = "x-ms-blob-public-access" ) @@ -426,28 +433,22 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { } // SetPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx -func (b BlobStorageClient) SetPermissions(container string, access ContainerAccessType) error { +func (b BlobStorageClient) SetPermissions(container string, accessOptions ContainerAccessOptions) error { params := url.Values{"restype": {"container"}, "comp": {"acl"}} - return b.setPermissionsCommon(container, access, params) -} - -// SetPermissions sets up container permissions (with timeout in seconds) as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx -func (b BlobStorageClient) SetPermissionsWithTimeout(container string, access ContainerAccessType, timeout int) error { - params := url.Values{"restype": {"container"}, - "comp": {"acl"}, - "timeout": {strconv.Itoa(timeout)}} - - return b.setPermissionsCommon(container, access, params) -} + if accessOptions.Timeout > 0 { + params.Add("timeout", strconv.Itoa(accessOptions.Timeout)) + } -// setPermissionsCommon sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx -func (b BlobStorageClient) setPermissionsCommon(container string, access ContainerAccessType, params url.Values) error { uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) headers := b.client.getStandardHeaders() - if access != "" { - headers[ContainerAccessHeader] = string(access) + if accessOptions.ContainerAccess != "" { + headers[ContainerAccessHeader] = string(accessOptions.ContainerAccess) + } + + if accessOptions.LeaseID != "" { + //headers[LeaseID] = accessOptions.LeaseID } verb := "PUT" From 5b3af804c72c54d90d9d6a6bf7261f6bc0a7dcdd Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Tue, 18 Oct 2016 18:45:02 +1100 Subject: [PATCH 10/31] adding in SAS part of permissions --- storage/blob.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index 9bf4a129fe19..974d6918611a 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -256,6 +256,11 @@ const ( blobCopyStatusFailed = "failed" ) +// lease constants. +const ( + leaseID = "x-ms-lease-id" +) + // BlockListType is used to filter out types of blocks in a Get Blocks List call // for a block blob. // @@ -291,6 +296,15 @@ type ContainerAccessOptions struct { LeaseID string } +type ShareAccessSignatureDetails struct { + Id string + StartTime time.Time + EndTime time.Time + CanRead bool + CanWrite bool + CanDelete bool +} + const ( ContainerAccessHeader string = "x-ms-blob-public-access" ) @@ -448,7 +462,7 @@ func (b BlobStorageClient) SetPermissions(container string, accessOptions Contai } if accessOptions.LeaseID != "" { - //headers[LeaseID] = accessOptions.LeaseID + headers[leaseID] = accessOptions.LeaseID } verb := "PUT" From 63cb499277a39a3c86411e3eb2e431bacf45a4ea Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Wed, 19 Oct 2016 08:07:19 +1100 Subject: [PATCH 11/31] Generate XML payload for setting permissions --- storage/blob.go | 68 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 974d6918611a..899ccf3be8e1 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -296,13 +296,19 @@ type ContainerAccessOptions struct { LeaseID string } -type ShareAccessSignatureDetails struct { - Id string - StartTime time.Time - EndTime time.Time - CanRead bool - CanWrite bool - CanDelete bool +// Access Polcity details used for generating SharedAccessSignatures`` +type AccessPolicyDetails struct { + ID string + StartTime time.Time + ExpiryTime time.Time + CanRead bool + CanWrite bool + CanDelete bool +} + +type ContainerPermissions struct { + AccessOptions ContainerAccessOptions + AccessPolicy AccessPolicyDetails } const ( @@ -446,27 +452,30 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { return false, err } -// SetPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx -func (b BlobStorageClient) SetPermissions(container string, accessOptions ContainerAccessOptions) error { +// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx +func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) error { params := url.Values{"restype": {"container"}, "comp": {"acl"}} - if accessOptions.Timeout > 0 { - params.Add("timeout", strconv.Itoa(accessOptions.Timeout)) + if containerPermissions.AccessOptions.Timeout > 0 { + params.Add("timeout", strconv.Itoa(containerPermissions.AccessOptions.Timeout)) } uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) headers := b.client.getStandardHeaders() - if accessOptions.ContainerAccess != "" { - headers[ContainerAccessHeader] = string(accessOptions.ContainerAccess) + if containerPermissions.AccessOptions.ContainerAccess != "" { + headers[ContainerAccessHeader] = string(containerPermissions.AccessOptions.ContainerAccess) } - if accessOptions.LeaseID != "" { - headers[leaseID] = accessOptions.LeaseID + if containerPermissions.AccessOptions.LeaseID != "" { + headers[leaseID] = containerPermissions.AccessOptions.LeaseID } + // generate the XML for the SharedAccessSignature if required. + accessPolicyXML := b.generateAccessPolicy(containerPermissions.AccessPolicy) + verb := "PUT" - resp, err := b.client.exec(verb, uri, headers, nil) + resp, err := b.client.exec(verb, uri, headers, strings.NewReader(accessPolicyXML)) if err != nil { return err } @@ -482,6 +491,33 @@ func (b BlobStorageClient) SetPermissions(container string, accessOptions Contai return nil } +// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. +// TODO(kpfaulkner) find better way to generate the XML. This is clunky. +func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) string { + s := `` + s += fmt.Sprintf("%s", accessPolicy.ID) + s += "" + + permissions := "" + if accessPolicy.CanRead { + permissions += "r" + } + + if accessPolicy.CanWrite { + permissions += "w" + } + + if accessPolicy.CanDelete { + permissions += "d" + } + + s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02T15:04:05-0700")) + s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02T15:04:05-0700")) + s += fmt.Sprintf("%s", permissions) + s += `` + return s +} + // DeleteContainer deletes the container with given name on the storage // account. If the container does not exist returns error. // From 309d47761f5e7f51aef825db692a66b6fc42b776 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Wed, 19 Oct 2016 18:27:13 +1100 Subject: [PATCH 12/31] permissions working unsure about access policy: --- storage/blob.go | 59 ++++++++++++++++++++++++++++++++--------------- storage/client.go | 3 +++ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 899ccf3be8e1..d882b8ed6c57 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "strconv" @@ -311,6 +312,13 @@ type ContainerPermissions struct { AccessPolicy AccessPolicyDetails } +func NewContainerPermissions() ContainerPermissions { + perms := ContainerPermissions{} + perms.AccessOptions = ContainerAccessOptions{} + + return perms +} + const ( ContainerAccessHeader string = "x-ms-blob-public-access" ) @@ -474,8 +482,16 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe // generate the XML for the SharedAccessSignature if required. accessPolicyXML := b.generateAccessPolicy(containerPermissions.AccessPolicy) - verb := "PUT" - resp, err := b.client.exec(verb, uri, headers, strings.NewReader(accessPolicyXML)) + var resp *storageResponse + var err error + if accessPolicyXML != "" { + log.Printf("XML is %s", accessPolicyXML) + headers["Content-Length"] = fmt.Sprintf("%v", len(accessPolicyXML)) + resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML)) + } else { + resp, err = b.client.exec("PUT", uri, headers, nil) + } + if err != nil { return err } @@ -494,28 +510,33 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe // generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. // TODO(kpfaulkner) find better way to generate the XML. This is clunky. func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) string { - s := `` - s += fmt.Sprintf("%s", accessPolicy.ID) - s += "" - permissions := "" - if accessPolicy.CanRead { - permissions += "r" - } + if accessPolicy.ID != "" { + s := `` + s += fmt.Sprintf("%s", accessPolicy.ID) + s += "" - if accessPolicy.CanWrite { - permissions += "w" - } + permissions := "" + if accessPolicy.CanRead { + permissions += "r" + } + + if accessPolicy.CanWrite { + permissions += "w" + } + + if accessPolicy.CanDelete { + permissions += "d" + } - if accessPolicy.CanDelete { - permissions += "d" + s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02")) + s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02")) + s += fmt.Sprintf("%s", permissions) + s += `` + return s } - s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02T15:04:05-0700")) - s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02T15:04:05-0700")) - s += fmt.Sprintf("%s", permissions) - s += `` - return s + return "" } // DeleteContainer deletes the container with given name on the storage diff --git a/storage/client.go b/storage/client.go index 2816e03ec6c9..5aa3a3a25849 100644 --- a/storage/client.go +++ b/storage/client.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "net/http" "net/url" "regexp" @@ -245,6 +246,8 @@ func (c Client) getAuthorizationHeader(verb, url string, headers map[string]stri } canonicalizedString := c.buildCanonicalizedString(verb, headers, canonicalizedResource) + + log.Printf("canonical string %s: ", canonicalizedString) return c.createAuthorizationHeader(canonicalizedString), nil } From f9a0fd17da70ec2a8dd4ce737aeccde883b73dd9 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Thu, 20 Oct 2016 17:38:30 +1100 Subject: [PATCH 13/31] working on XML deserialization, WIP --- storage/blob.go | 64 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index d882b8ed6c57..708aedae1d65 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" "strconv" @@ -297,7 +296,7 @@ type ContainerAccessOptions struct { LeaseID string } -// Access Polcity details used for generating SharedAccessSignatures`` +// Access Policy details used for generating SharedAccessSignatures type AccessPolicyDetails struct { ID string StartTime time.Time @@ -312,6 +311,30 @@ type ContainerPermissions struct { AccessPolicy AccessPolicyDetails } +type AccessPolicyDetailsResponse struct { + StartTime string `xml:"Start"` + ExpiryTime string `xml:"Expiry"` + Permission string `xml:"Permission"` +} + +type SignedIdentifierResponse struct { + ID string `xml:"Id"` + AccessPolicy AccessPolicyDetailsResponse `xml:"AccessPolicy"` +} + +type SignedIdentifiers struct { + SignedIdentifiers []SignedIdentifierResponse `xml:"SignedIdentifier"` +} + +type AccessPolicyResponse struct { + SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"` +} + +type ContainerAccessResponse struct { + ContainerAccess ContainerAccessType + AccessPolicy AccessPolicyResponse +} + func NewContainerPermissions() ContainerPermissions { perms := ContainerPermissions{} perms.AccessOptions = ContainerAccessOptions{} @@ -485,7 +508,6 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe var resp *storageResponse var err error if accessPolicyXML != "" { - log.Printf("XML is %s", accessPolicyXML) headers["Content-Length"] = fmt.Sprintf("%v", len(accessPolicyXML)) resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML)) } else { @@ -503,10 +525,44 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe return errors.New("Unable to set permissions") } } - return nil } +// GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx +func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (*ContainerAccessResponse, error) { + params := url.Values{"restype": {"container"}, + "comp": {"acl"}} + + if timeout > 0 { + params.Add("timeout", strconv.Itoa(timeout)) + } + + uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) + headers := b.client.getStandardHeaders() + + if leaseID != "" { + headers[leaseID] = leaseID + } + + resp, err := b.client.exec("GET", uri, headers, nil) + + if err != nil { + return nil, err + } + + defer resp.body.Close() + var out AccessPolicyResponse + err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) + if err != nil { + return nil, err + } + + response := ContainerAccessResponse{} + response.AccessPolicy = out + //response.ContainerAccess = + return nil, nil +} + // generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. // TODO(kpfaulkner) find better way to generate the XML. This is clunky. func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) string { From 4184e8d53798454efdfd7e2bd5c3e1aa6c29e1a4 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Thu, 20 Oct 2016 19:50:00 +1100 Subject: [PATCH 14/31] GetContainerPermission returning permissions correctly. Check for refactoring --- storage/blob.go | 21 +++++++++++++++------ storage/client.go | 3 --- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 708aedae1d65..4550d531e42a 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "strconv" @@ -529,7 +530,8 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe } // GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx -func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (*ContainerAccessResponse, error) { +// Returns permissionResponse which is combined permissions and AccessPolicy +func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (permissionResponse *ContainerAccessResponse, err error) { params := url.Values{"restype": {"container"}, "comp": {"acl"}} @@ -545,11 +547,17 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int } resp, err := b.client.exec("GET", uri, headers, nil) - if err != nil { return nil, err } + for k, v := range resp.headers { + log.Printf("%s : %s", k, v) + } + + // containerAccess. Blob, Container, empty + containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) + defer resp.body.Close() var out AccessPolicyResponse err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) @@ -557,10 +565,11 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int return nil, err } - response := ContainerAccessResponse{} - response.AccessPolicy = out - //response.ContainerAccess = - return nil, nil + permissionResponse = &ContainerAccessResponse{} + permissionResponse.AccessPolicy = out + permissionResponse.ContainerAccess = ContainerAccessType(containerAccess) + + return permissionResponse, nil } // generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. diff --git a/storage/client.go b/storage/client.go index 5aa3a3a25849..2816e03ec6c9 100644 --- a/storage/client.go +++ b/storage/client.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/url" "regexp" @@ -246,8 +245,6 @@ func (c Client) getAuthorizationHeader(verb, url string, headers map[string]stri } canonicalizedString := c.buildCanonicalizedString(verb, headers, canonicalizedResource) - - log.Printf("canonical string %s: ", canonicalizedString) return c.createAuthorizationHeader(canonicalizedString), nil } From 907bd6c4639e86ddcddd65acac5ddbd410b109cb Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 21 Oct 2016 07:56:48 +1100 Subject: [PATCH 15/31] containerACL tests in progress --- storage/blob.go | 32 ++++++++++++++------------- storage/blob_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 33e88462416c..24e6e9dc200f 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -309,7 +309,7 @@ type ContainerAccessOptions struct { LeaseID string } -// Access Policy details used for generating SharedAccessSignatures +// AccessPolicyDetails are used for SETTING policies type AccessPolicyDetails struct { ID string StartTime time.Time @@ -319,40 +319,40 @@ type AccessPolicyDetails struct { CanDelete bool } +// ContainerPermissions is used when setting permissions and Access Policies for containers. type ContainerPermissions struct { AccessOptions ContainerAccessOptions AccessPolicy AccessPolicyDetails } +// AccessPolicyDetailsResponse has specifics about an access policy type AccessPolicyDetailsResponse struct { - StartTime string `xml:"Start"` - ExpiryTime string `xml:"Expiry"` - Permission string `xml:"Permission"` + StartTime time.Time `xml:"Start"` + ExpiryTime time.Time `xml:"Expiry"` + Permission string `xml:"Permission"` } +// SignedIdentifierResponse is a wrapper for a specific policy type SignedIdentifierResponse struct { ID string `xml:"Id"` AccessPolicy AccessPolicyDetailsResponse `xml:"AccessPolicy"` } +// SignedIdentifiers part of the response from GetPermissions call. type SignedIdentifiers struct { SignedIdentifiers []SignedIdentifierResponse `xml:"SignedIdentifier"` } +// AccessPolicyResponse is the response type from the GetPermissions call. type AccessPolicyResponse struct { SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"` } +// ContainerAccessResponse is returned for the GetContainerPermissions function. +// This contains both the permission and access policy for the container. type ContainerAccessResponse struct { ContainerAccess ContainerAccessType - AccessPolicy AccessPolicyResponse -} - -func NewContainerPermissions() ContainerPermissions { - perms := ContainerPermissions{} - perms.AccessOptions = ContainerAccessOptions{} - - return perms + AccessPolicy SignedIdentifiers } const ( @@ -542,6 +542,8 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe } // GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx +// If timeout is 0 then it will not be passed to Azure +// leaseID will only be passed to Azure if populated // Returns permissionResponse which is combined permissions and AccessPolicy func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (permissionResponse *ContainerAccessResponse, err error) { params := url.Values{"restype": {"container"}, @@ -578,7 +580,7 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int } permissionResponse = &ContainerAccessResponse{} - permissionResponse.AccessPolicy = out + permissionResponse.AccessPolicy = out.SignedIdentifiersList permissionResponse.ContainerAccess = ContainerAccessType(containerAccess) return permissionResponse, nil @@ -606,8 +608,8 @@ func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails permissions += "d" } - s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02")) - s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02")) + s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02T15:04:05Z")) + s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z")) s += fmt.Sprintf("%s", permissions) s += `` return s diff --git a/storage/blob_test.go b/storage/blob_test.go index 857181955afd..2eb5e09191ac 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -678,6 +678,57 @@ func (s *StorageBlobSuite) TestSetBlobProperties(c *chk.C) { c.Check(mPut.ContentLanguage, chk.Equals, props.ContentLanguage) } +func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccessType, + timeout int, leaseID string, ID string, canRead bool, + canWrite bool, canDelete bool) ContainerPermissions { + perms := ContainerPermissions{} + perms.AccessOptions.ContainerAccess = accessType + perms.AccessOptions.Timeout = timeout + perms.AccessOptions.LeaseID = leaseID + perms.AccessPolicy.ID = ID + perms.AccessPolicy.StartTime = time.Now() + perms.AccessPolicy.ExpiryTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) + perms.AccessPolicy.CanRead = canRead + perms.AccessPolicy.CanWrite = canWrite + perms.AccessPolicy.CanDelete = canDelete + return perms +} + +func (s *StorageBlobSuite) TestSetContainerPermissionsSuccessfully(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + perms := s.createContainerPermissions(ContainerAccessTypeBlob, 0, "", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTa=", true, true, true) + + err := cli.SetContainerPermissions(cnt, perms) + c.Assert(err, chk.IsNil) +} + +func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsSuccessfully(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + perms := s.createContainerPermissions(ContainerAccessTypeBlob, 0, "", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTa=", true, true, true) + + err := cli.SetContainerPermissions(cnt, perms) + c.Assert(err, chk.IsNil) + + returnedPerms, err := cli.GetContainerPermissions(cnt, 0, "") + c.Assert(err, chk.IsNil) + + // check container permissions itself. + c.Assert(perms.AccessOptions.ContainerAccess, chk.Equals, returnedPerms.ContainerAccess) + + // now check policy set. + c.Assert(perms.AccessOptions.ContainerAccess, chk.Equals, returnedPerms.ContainerAccess) + +} func (s *StorageBlobSuite) TestAcquireLeaseWithNoProposedLeaseID(c *chk.C) { cli := getBlobClient(c) cnt := randContainer() From 362ef1dea6942f0c29a6ea92cfa711544e6846d3 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 21 Oct 2016 18:04:19 +1100 Subject: [PATCH 16/31] SetContainerPermission and GetContainerPermission tests --- storage/blob_test.go | 74 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/storage/blob_test.go b/storage/blob_test.go index 2eb5e09191ac..29b30f3fda61 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -685,15 +685,32 @@ func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccess perms.AccessOptions.ContainerAccess = accessType perms.AccessOptions.Timeout = timeout perms.AccessOptions.LeaseID = leaseID - perms.AccessPolicy.ID = ID - perms.AccessPolicy.StartTime = time.Now() - perms.AccessPolicy.ExpiryTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) - perms.AccessPolicy.CanRead = canRead - perms.AccessPolicy.CanWrite = canWrite - perms.AccessPolicy.CanDelete = canDelete + + if ID != "" { + perms.AccessPolicy.ID = ID + perms.AccessPolicy.StartTime = time.Now() + perms.AccessPolicy.ExpiryTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) + perms.AccessPolicy.CanRead = canRead + perms.AccessPolicy.CanWrite = canWrite + perms.AccessPolicy.CanDelete = canDelete + } + return perms } +func (s *StorageBlobSuite) TestSetContainerPermissionsWithTimeoutSuccessfully(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + perms := s.createContainerPermissions(ContainerAccessTypeBlob, 30, "", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTa=", true, true, true) + + err := cli.SetContainerPermissions(cnt, perms) + c.Assert(err, chk.IsNil) +} + func (s *StorageBlobSuite) TestSetContainerPermissionsSuccessfully(c *chk.C) { cli := getBlobClient(c) cnt := randContainer() @@ -723,12 +740,53 @@ func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsSuccessfully(c *chk c.Assert(err, chk.IsNil) // check container permissions itself. - c.Assert(perms.AccessOptions.ContainerAccess, chk.Equals, returnedPerms.ContainerAccess) + c.Assert(returnedPerms.ContainerAccess, chk.Equals, perms.AccessOptions.ContainerAccess) // now check policy set. - c.Assert(perms.AccessOptions.ContainerAccess, chk.Equals, returnedPerms.ContainerAccess) + c.Assert(1, chk.Equals, len(returnedPerms.AccessPolicy.SignedIdentifiers)) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].ID, chk.Equals, perms.AccessPolicy.ID) + // test timestamps down the minutes + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.StartTime.Format("2006-01-02T15:04:05Z"), chk.Equals, perms.AccessPolicy.StartTime.Format("2006-01-02T15:04:05Z")) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z"), chk.Equals, perms.AccessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z")) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.Permission, chk.Equals, "rwd") } + +func (s *StorageBlobSuite) TestSetContainerPermissionsOnlySuccessfully(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + perms := s.createContainerPermissions(ContainerAccessTypeBlob, 0, "", "", true, true, true) + + err := cli.SetContainerPermissions(cnt, perms) + c.Assert(err, chk.IsNil) +} + +func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsOnlySuccessfully(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + perms := s.createContainerPermissions(ContainerAccessTypeBlob, 0, "", "", true, true, true) + + err := cli.SetContainerPermissions(cnt, perms) + c.Assert(err, chk.IsNil) + + returnedPerms, err := cli.GetContainerPermissions(cnt, 0, "") + c.Assert(err, chk.IsNil) + + // check container permissions itself. + c.Assert(returnedPerms.ContainerAccess, chk.Equals, perms.AccessOptions.ContainerAccess) + + // now check there are NO policies set + c.Assert(0, chk.Equals, len(returnedPerms.AccessPolicy.SignedIdentifiers)) +} + func (s *StorageBlobSuite) TestAcquireLeaseWithNoProposedLeaseID(c *chk.C) { cli := getBlobClient(c) cnt := randContainer() From 5bf4861a921a78cee2e914c4557771a85bfb26a6 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 21 Oct 2016 22:17:37 +1100 Subject: [PATCH 17/31] removing some debugging --- storage/blob.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index bdeb576dda09..be7fac9b8639 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" "strconv" @@ -565,10 +564,6 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int return nil, err } - for k, v := range resp.headers { - log.Printf("%s : %s", k, v) - } - // containerAccess. Blob, Container, empty containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) From 831ca62eb4b4a3133869c799fe09d4d499459b2e Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Sat, 22 Oct 2016 07:30:30 +1100 Subject: [PATCH 18/31] formatting fix causing lint error --- storage/blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index be7fac9b8639..9d3a635a5aa3 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -354,6 +354,7 @@ type ContainerAccessResponse struct { AccessPolicy SignedIdentifiers } +// ContainerAccessHeader references header used when setting/getting container ACL const ( ContainerAccessHeader string = "x-ms-blob-public-access" ) @@ -771,7 +772,6 @@ func (b BlobStorageClient) leaseCommonPut(container string, name string, headers } // AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx - // returns leaseID acquired func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (returnedLeaseID string, err error) { headers := b.client.getStandardHeaders() From 522dfdc2560672682d84595f75903905619198bb Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Thu, 27 Oct 2016 08:27:29 +1100 Subject: [PATCH 19/31] changes based on Mcardosos' feedback. --- storage/blob.go | 10 ++++++---- storage/blob_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 9d3a635a5aa3..af74b417162a 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -496,10 +496,12 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { return false, err } -// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391(d=printer).aspx +// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391.aspx func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) error { - params := url.Values{"restype": {"container"}, - "comp": {"acl"}} + params := url.Values{ + "restype": {"container"}, + "comp": {"acl"}, + } if containerPermissions.AccessOptions.Timeout > 0 { params.Add("timeout", strconv.Itoa(containerPermissions.AccessOptions.Timeout)) @@ -521,7 +523,7 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe var resp *storageResponse var err error if accessPolicyXML != "" { - headers["Content-Length"] = fmt.Sprintf("%v", len(accessPolicyXML)) + headers["Content-Length"] = strconv.Itoa(len(accessPolicyXML)) resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML)) } else { resp, err = b.client.exec("PUT", uri, headers, nil) diff --git a/storage/blob_test.go b/storage/blob_test.go index 29b30f3fda61..e217733de8de 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -689,7 +689,7 @@ func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccess if ID != "" { perms.AccessPolicy.ID = ID perms.AccessPolicy.StartTime = time.Now() - perms.AccessPolicy.ExpiryTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) + perms.AccessPolicy.ExpiryTime = time.Now().Add(time.Hour * 10) perms.AccessPolicy.CanRead = canRead perms.AccessPolicy.CanWrite = canWrite perms.AccessPolicy.CanDelete = canDelete @@ -743,12 +743,12 @@ func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsSuccessfully(c *chk c.Assert(returnedPerms.ContainerAccess, chk.Equals, perms.AccessOptions.ContainerAccess) // now check policy set. - c.Assert(1, chk.Equals, len(returnedPerms.AccessPolicy.SignedIdentifiers)) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers, chk.HasLen, 1) c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].ID, chk.Equals, perms.AccessPolicy.ID) // test timestamps down the minutes - c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.StartTime.Format("2006-01-02T15:04:05Z"), chk.Equals, perms.AccessPolicy.StartTime.Format("2006-01-02T15:04:05Z")) - c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z"), chk.Equals, perms.AccessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z")) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.StartTime.Format(time.RFC1123), chk.Equals, perms.AccessPolicy.StartTime.Format(time.RFC1123)) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.ExpiryTime.Format(time.RFC1123), chk.Equals, perms.AccessPolicy.ExpiryTime.Format(time.RFC1123)) c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.Permission, chk.Equals, "rwd") } @@ -784,7 +784,7 @@ func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsOnlySuccessfully(c c.Assert(returnedPerms.ContainerAccess, chk.Equals, perms.AccessOptions.ContainerAccess) // now check there are NO policies set - c.Assert(0, chk.Equals, len(returnedPerms.AccessPolicy.SignedIdentifiers)) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers, chk.HasLen, 0) } func (s *StorageBlobSuite) TestAcquireLeaseWithNoProposedLeaseID(c *chk.C) { From f22ad3db44e0ae7e7724a8bf45c9fb5186361e6f Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 11:54:33 +1100 Subject: [PATCH 20/31] marshalling working --- storage/blob.go | 98 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index af74b417162a..0766df92a0bf 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "net/url" "strconv" @@ -324,26 +325,27 @@ type ContainerPermissions struct { AccessPolicy AccessPolicyDetails } -// AccessPolicyDetailsResponse has specifics about an access policy -type AccessPolicyDetailsResponse struct { +// AccessPolicyDetailsXML has specifics about an access policy +// annotated with XML details. +type AccessPolicyDetailsXML struct { StartTime time.Time `xml:"Start"` ExpiryTime time.Time `xml:"Expiry"` Permission string `xml:"Permission"` } -// SignedIdentifierResponse is a wrapper for a specific policy -type SignedIdentifierResponse struct { - ID string `xml:"Id"` - AccessPolicy AccessPolicyDetailsResponse `xml:"AccessPolicy"` +// SignedIdentifier is a wrapper for a specific policy +type SignedIdentifier struct { + ID string `xml:"Id"` + AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"` } // SignedIdentifiers part of the response from GetPermissions call. type SignedIdentifiers struct { - SignedIdentifiers []SignedIdentifierResponse `xml:"SignedIdentifier"` + SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"` } -// AccessPolicyResponse is the response type from the GetPermissions call. -type AccessPolicyResponse struct { +// AccessPolicy is the response type from the GetPermissions call. +type AccessPolicy struct { SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"` } @@ -518,10 +520,12 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe } // generate the XML for the SharedAccessSignature if required. - accessPolicyXML := b.generateAccessPolicy(containerPermissions.AccessPolicy) + accessPolicyXML, err := b.generateAccessPolicy(containerPermissions.AccessPolicy) + if err != nil { + return err + } var resp *storageResponse - var err error if accessPolicyXML != "" { headers["Content-Length"] = strconv.Itoa(len(accessPolicyXML)) resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML)) @@ -571,7 +575,7 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) defer resp.body.Close() - var out AccessPolicyResponse + var out AccessPolicy err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) if err != nil { return nil, err @@ -584,36 +588,62 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int return permissionResponse, nil } -// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. -// TODO(kpfaulkner) find better way to generate the XML. This is clunky. -func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) string { +func (b BlobStorageClient) generatePermissions(accessPolicy AccessPolicyDetails) (permissions string) { + // generate the permissions string (rwd). + // still want the end user API to have bool flags. + permissions = "" - if accessPolicy.ID != "" { - s := `` - s += fmt.Sprintf("%s", accessPolicy.ID) - s += "" + if accessPolicy.CanRead { + permissions += "r" + } - permissions := "" - if accessPolicy.CanRead { - permissions += "r" - } + if accessPolicy.CanWrite { + permissions += "w" + } - if accessPolicy.CanWrite { - permissions += "w" - } + if accessPolicy.CanDelete { + permissions += "d" + } - if accessPolicy.CanDelete { - permissions += "d" + return permissions +} + +// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the +// AccessPolicy struct which will get converted to XML. +func (b BlobStorageClient) convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) (accessPolicyToMarshal AccessPolicy) { + accessPolicyToMarshal = AccessPolicy{} + accessPolicyToMarshal.SignedIdentifiersList = SignedIdentifiers{} + accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers = []SignedIdentifier{} + si := SignedIdentifier{} + si.ID = accessPolicy.ID + si.AccessPolicy = AccessPolicyDetailsXML{} + si.AccessPolicy.StartTime = accessPolicy.StartTime + si.AccessPolicy.ExpiryTime = accessPolicy.ExpiryTime + permissions := b.generatePermissions(accessPolicy) + si.AccessPolicy.Permission = permissions + accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers = append(accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers, si) + return accessPolicyToMarshal +} + +// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. +func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) (accessPolicyXML string, err error) { + + if accessPolicy.ID != "" { + ap := b.convertAccessPolicyToXMLStructs(accessPolicy) + body, _, err := xmlMarshal(ap.SignedIdentifiersList) + if err != nil { + return "", err } - s += fmt.Sprintf("%s", accessPolicy.StartTime.Format("2006-01-02T15:04:05Z")) - s += fmt.Sprintf("%s", accessPolicy.ExpiryTime.Format("2006-01-02T15:04:05Z")) - s += fmt.Sprintf("%s", permissions) - s += `` - return s + xmlByteArray, err := ioutil.ReadAll(body) + if err != nil { + return "", err + } + accessPolicyXML = string(xmlByteArray) + return accessPolicyXML, nil } - return "" + return "", errors.New("Unable to generate AccessPolicy XML") } // DeleteContainer deletes the container with given name on the storage From bb22d80f239d915bd94264a60e5250f91f670ee1 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 12:00:34 +1100 Subject: [PATCH 21/31] fixing test --- storage/blob_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/blob_test.go b/storage/blob_test.go index e217733de8de..47cc2c406564 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -688,8 +688,8 @@ func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccess if ID != "" { perms.AccessPolicy.ID = ID - perms.AccessPolicy.StartTime = time.Now() - perms.AccessPolicy.ExpiryTime = time.Now().Add(time.Hour * 10) + perms.AccessPolicy.StartTime = time.Now().UTC() + perms.AccessPolicy.ExpiryTime = time.Now().UTC().Add(time.Hour * 10) perms.AccessPolicy.CanRead = canRead perms.AccessPolicy.CanWrite = canWrite perms.AccessPolicy.CanDelete = canDelete From 21f854e14100a2dccfc44a11061f2a6602730e68 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 12:06:36 +1100 Subject: [PATCH 22/31] test tweak --- storage/blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/blob.go b/storage/blob.go index 0766df92a0bf..6b833fbef540 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -643,7 +643,7 @@ func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails return accessPolicyXML, nil } - return "", errors.New("Unable to generate AccessPolicy XML") + return "", nil } // DeleteContainer deletes the container with given name on the storage From 7287de88ecfb31bcbfc86cbba17a0d0fc2f9e6bb Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 12:37:10 +1100 Subject: [PATCH 23/31] debugging tests working fine locally --- storage/blob.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/blob.go b/storage/blob.go index 6b833fbef540..edbf77c52e5f 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "net/http" "net/url" "strconv" @@ -521,6 +522,8 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe // generate the XML for the SharedAccessSignature if required. accessPolicyXML, err := b.generateAccessPolicy(containerPermissions.AccessPolicy) + log.Printf("AccessPolicyXML is %s", accessPolicyXML) + if err != nil { return err } From 84c0f6ced1177df8cac5d95be7a8bc5036ed4350 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 13:15:44 +1100 Subject: [PATCH 24/31] rounding of access policy times --- storage/blob_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/blob_test.go b/storage/blob_test.go index 47cc2c406564..eeb8353cb750 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -688,8 +688,8 @@ func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccess if ID != "" { perms.AccessPolicy.ID = ID - perms.AccessPolicy.StartTime = time.Now().UTC() - perms.AccessPolicy.ExpiryTime = time.Now().UTC().Add(time.Hour * 10) + perms.AccessPolicy.StartTime = time.Now().UTC().Round(time.Second) + perms.AccessPolicy.ExpiryTime = time.Now().UTC().Add(time.Hour * 10).Round(time.Second) perms.AccessPolicy.CanRead = canRead perms.AccessPolicy.CanWrite = canWrite perms.AccessPolicy.CanDelete = canDelete From 0df936cb0defd9816fc89d5b1382aac70b89bbf4 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 28 Oct 2016 13:20:54 +1100 Subject: [PATCH 25/31] force rounding of dates for policy --- storage/blob.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index edbf77c52e5f..61c9641531f5 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/url" "strconv" @@ -522,8 +521,6 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe // generate the XML for the SharedAccessSignature if required. accessPolicyXML, err := b.generateAccessPolicy(containerPermissions.AccessPolicy) - log.Printf("AccessPolicyXML is %s", accessPolicyXML) - if err != nil { return err } @@ -620,8 +617,8 @@ func (b BlobStorageClient) convertAccessPolicyToXMLStructs(accessPolicy AccessPo si := SignedIdentifier{} si.ID = accessPolicy.ID si.AccessPolicy = AccessPolicyDetailsXML{} - si.AccessPolicy.StartTime = accessPolicy.StartTime - si.AccessPolicy.ExpiryTime = accessPolicy.ExpiryTime + si.AccessPolicy.StartTime = accessPolicy.StartTime.Round(time.Second) + si.AccessPolicy.ExpiryTime = accessPolicy.ExpiryTime.Round(time.Second) permissions := b.generatePermissions(accessPolicy) si.AccessPolicy.Permission = permissions accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers = append(accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers, si) From 24913ed835cdf41e0d787ffd1964483c5ea8e222 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Tue, 1 Nov 2016 08:05:56 +1100 Subject: [PATCH 26/31] changes based on review --- storage/blob.go | 118 ++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 61c9641531f5..d6b2b48efb65 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -520,7 +520,7 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe } // generate the XML for the SharedAccessSignature if required. - accessPolicyXML, err := b.generateAccessPolicy(containerPermissions.AccessPolicy) + accessPolicyXML, err := generateAccessPolicy(containerPermissions.AccessPolicy) if err != nil { return err } @@ -588,64 +588,6 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int return permissionResponse, nil } -func (b BlobStorageClient) generatePermissions(accessPolicy AccessPolicyDetails) (permissions string) { - // generate the permissions string (rwd). - // still want the end user API to have bool flags. - permissions = "" - - if accessPolicy.CanRead { - permissions += "r" - } - - if accessPolicy.CanWrite { - permissions += "w" - } - - if accessPolicy.CanDelete { - permissions += "d" - } - - return permissions -} - -// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the -// AccessPolicy struct which will get converted to XML. -func (b BlobStorageClient) convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) (accessPolicyToMarshal AccessPolicy) { - accessPolicyToMarshal = AccessPolicy{} - accessPolicyToMarshal.SignedIdentifiersList = SignedIdentifiers{} - accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers = []SignedIdentifier{} - si := SignedIdentifier{} - si.ID = accessPolicy.ID - si.AccessPolicy = AccessPolicyDetailsXML{} - si.AccessPolicy.StartTime = accessPolicy.StartTime.Round(time.Second) - si.AccessPolicy.ExpiryTime = accessPolicy.ExpiryTime.Round(time.Second) - permissions := b.generatePermissions(accessPolicy) - si.AccessPolicy.Permission = permissions - accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers = append(accessPolicyToMarshal.SignedIdentifiersList.SignedIdentifiers, si) - return accessPolicyToMarshal -} - -// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. -func (b BlobStorageClient) generateAccessPolicy(accessPolicy AccessPolicyDetails) (accessPolicyXML string, err error) { - - if accessPolicy.ID != "" { - ap := b.convertAccessPolicyToXMLStructs(accessPolicy) - body, _, err := xmlMarshal(ap.SignedIdentifiersList) - if err != nil { - return "", err - } - - xmlByteArray, err := ioutil.ReadAll(body) - if err != nil { - return "", err - } - accessPolicyXML = string(xmlByteArray) - return accessPolicyXML, nil - } - - return "", nil -} - // DeleteContainer deletes the container with given name on the storage // account. If the container does not exist returns error. // @@ -1482,3 +1424,61 @@ func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, sig } return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") } + +func generatePermissions(accessPolicy AccessPolicyDetails) (permissions string) { + // generate the permissions string (rwd). + // still want the end user API to have bool flags. + permissions = "" + + if accessPolicy.CanRead { + permissions += "r" + } + + if accessPolicy.CanWrite { + permissions += "w" + } + + if accessPolicy.CanDelete { + permissions += "d" + } + + return permissions +} + +// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the +// AccessPolicy struct which will get converted to XML. +func convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) SignedIdentifiers { + return SignedIdentifiers{ + SignedIdentifiers: []SignedIdentifier{ + { + ID: accessPolicy.ID, + AccessPolicy: AccessPolicyDetailsXML{ + StartTime: accessPolicy.StartTime.Round(time.Second), + ExpiryTime: accessPolicy.ExpiryTime.Round(time.Second), + Permission: generatePermissions(accessPolicy), + }, + }, + }, + } +} + +// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. +func generateAccessPolicy(accessPolicy AccessPolicyDetails) (accessPolicyXML string, err error) { + + if accessPolicy.ID != "" { + signedIdentifiers := convertAccessPolicyToXMLStructs(accessPolicy) + body, _, err := xmlMarshal(signedIdentifiers) + if err != nil { + return "", err + } + + xmlByteArray, err := ioutil.ReadAll(body) + if err != nil { + return "", err + } + accessPolicyXML = string(xmlByteArray) + return accessPolicyXML, nil + } + + return "", nil +} From 6a9edd163c7664c1941c0369ddea9e05d086293a Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Wed, 2 Nov 2016 18:43:43 +1100 Subject: [PATCH 27/31] capturing potential error from closing body --- storage/blob.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index d6b2b48efb65..52b63e686783 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -499,7 +499,7 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { } // SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391.aspx -func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) error { +func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) (err error) { params := url.Values{ "restype": {"container"}, "comp": {"acl"}, @@ -538,7 +538,9 @@ func (b BlobStorageClient) SetContainerPermissions(container string, containerPe } if resp != nil { - defer resp.body.Close() + defer func() { + err = resp.body.Close() + }() if resp.statusCode != http.StatusOK { return errors.New("Unable to set permissions") @@ -574,7 +576,10 @@ func (b BlobStorageClient) GetContainerPermissions(container string, timeout int // containerAccess. Blob, Container, empty containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) - defer resp.body.Close() + defer func() { + err = resp.body.Close() + }() + var out AccessPolicy err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) if err != nil { From 3218bf1e4f6c765fa881c58d1eaebfd432136307 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 4 Nov 2016 12:47:23 +1100 Subject: [PATCH 28/31] change accesspolicy to convert to UTC --- storage/blob.go | 4 ++-- storage/blob_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 52b63e686783..9b9b74b6e3ef 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -1458,8 +1458,8 @@ func convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) SignedIde { ID: accessPolicy.ID, AccessPolicy: AccessPolicyDetailsXML{ - StartTime: accessPolicy.StartTime.Round(time.Second), - ExpiryTime: accessPolicy.ExpiryTime.Round(time.Second), + StartTime: accessPolicy.StartTime.UTC().Round(time.Nanosecond * 100), + ExpiryTime: accessPolicy.ExpiryTime.UTC().Round(time.Nanosecond * 100), Permission: generatePermissions(accessPolicy), }, }, diff --git a/storage/blob_test.go b/storage/blob_test.go index eeb8353cb750..1a05bb80a48a 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -42,7 +42,7 @@ func (s *StorageBlobSuite) Test_blobSASStringToSign(c *chk.C) { out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP") c.Assert(err, chk.IsNil) - c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n") + c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013f-08-15\n\n\n\n\n") } func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) { @@ -688,8 +688,8 @@ func (s *StorageBlobSuite) createContainerPermissions(accessType ContainerAccess if ID != "" { perms.AccessPolicy.ID = ID - perms.AccessPolicy.StartTime = time.Now().UTC().Round(time.Second) - perms.AccessPolicy.ExpiryTime = time.Now().UTC().Add(time.Hour * 10).Round(time.Second) + perms.AccessPolicy.StartTime = time.Now() + perms.AccessPolicy.ExpiryTime = time.Now().Add(time.Hour * 10) perms.AccessPolicy.CanRead = canRead perms.AccessPolicy.CanWrite = canWrite perms.AccessPolicy.CanDelete = canDelete From 5b8d080c8ba4fafbd6558d17d3a9b3f9ef0396d6 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 4 Nov 2016 16:32:13 +1100 Subject: [PATCH 29/31] reverting to rounding to second. --- storage/blob.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/blob.go b/storage/blob.go index 9b9b74b6e3ef..1d4c12b796bb 100644 --- a/storage/blob.go +++ b/storage/blob.go @@ -1458,8 +1458,8 @@ func convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) SignedIde { ID: accessPolicy.ID, AccessPolicy: AccessPolicyDetailsXML{ - StartTime: accessPolicy.StartTime.UTC().Round(time.Nanosecond * 100), - ExpiryTime: accessPolicy.ExpiryTime.UTC().Round(time.Nanosecond * 100), + StartTime: accessPolicy.StartTime.UTC().Round(time.Second), + ExpiryTime: accessPolicy.ExpiryTime.UTC().Round(time.Second), Permission: generatePermissions(accessPolicy), }, }, From ded4c386e2777f4798720520514bed65092842f8 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 4 Nov 2016 16:42:54 +1100 Subject: [PATCH 30/31] rounding times for tests so test reflects reality --- storage/blob_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/storage/blob_test.go b/storage/blob_test.go index 1a05bb80a48a..d24e8f2a92ab 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -746,9 +746,11 @@ func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsSuccessfully(c *chk c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers, chk.HasLen, 1) c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].ID, chk.Equals, perms.AccessPolicy.ID) - // test timestamps down the minutes - c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.StartTime.Format(time.RFC1123), chk.Equals, perms.AccessPolicy.StartTime.Format(time.RFC1123)) - c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.ExpiryTime.Format(time.RFC1123), chk.Equals, perms.AccessPolicy.ExpiryTime.Format(time.RFC1123)) + // test timestamps down the second + // rounding start/expiry time original perms since the returned perms would have been rounded. + // so need rounded vs rounded. + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.StartTime.Round(time.Second).Format(time.RFC1123), chk.Equals, perms.AccessPolicy.StartTime.Round(time.Second).Format(time.RFC1123)) + c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.ExpiryTime.Round(time.Second).Format(time.RFC1123), chk.Equals, perms.AccessPolicy.ExpiryTime.Round(time.Second).Format(time.RFC1123)) c.Assert(returnedPerms.AccessPolicy.SignedIdentifiers[0].AccessPolicy.Permission, chk.Equals, "rwd") } From c47bff47f2d0b2af26f8302d7b65c38b70bf2980 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Fri, 4 Nov 2016 16:50:12 +1100 Subject: [PATCH 31/31] typo --- storage/blob_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/blob_test.go b/storage/blob_test.go index d24e8f2a92ab..7b5793959b14 100644 --- a/storage/blob_test.go +++ b/storage/blob_test.go @@ -42,7 +42,7 @@ func (s *StorageBlobSuite) Test_blobSASStringToSign(c *chk.C) { out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP") c.Assert(err, chk.IsNil) - c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013f-08-15\n\n\n\n\n") + c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n") } func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) {