From 6dea7a96d5290a2ab79e898bd589d6f937f2b179 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Fri, 2 Jun 2017 16:20:53 -0700 Subject: [PATCH 1/7] Encryption at REST for files --- ChangeLog.txt | 2 + .../file/CloudFileServerEncryptionTests.java | 127 ++++++++++++++++++ .../azure/storage/blob/CloudAppendBlob.java | 5 +- .../azure/storage/blob/CloudBlob.java | 7 +- .../azure/storage/blob/CloudBlockBlob.java | 7 +- .../azure/storage/blob/CloudPageBlob.java | 5 +- .../azure/storage/core/BaseResponse.java | 10 ++ .../azure/storage/file/CloudFile.java | 6 + .../storage/file/CloudFileDirectory.java | 3 + .../storage/file/FileDirectoryProperties.java | 24 ++++ .../azure/storage/file/FileProperties.java | 27 +++- .../azure/storage/file/FileResponse.java | 4 + 12 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 microsoft-azure-storage-test/src/com/microsoft/azure/storage/file/CloudFileServerEncryptionTests.java diff --git a/ChangeLog.txt b/ChangeLog.txt index 67632a99d2320..7e77098aa4e73 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,3 +1,5 @@ +2017.XX.XX Version X.X.0 + * Added support for server side encryption for File Service. 2017.05.23 Version 5.2.0 * Fixed Exists() calls on Shares and Directories to now populate metadata. This was already being done for Files. * Changed blob constants to support up to 256 MB on put blob for block blobs. The default value for put blob threshold has also been updated to half of the maximum, or 128 MB currently. diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/file/CloudFileServerEncryptionTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/file/CloudFileServerEncryptionTests.java new file mode 100644 index 0000000000000..be815738036f8 --- /dev/null +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/file/CloudFileServerEncryptionTests.java @@ -0,0 +1,127 @@ +/** + * Copyright Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.microsoft.azure.storage.file; + + +import java.io.IOException; +import java.net.URISyntaxException; + +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestCompletedEvent; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.TestRunners.CloudTests; +import com.microsoft.azure.storage.TestRunners.DevFabricTests; +import com.microsoft.azure.storage.TestRunners.DevStoreTests; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category({ CloudTests.class, DevFabricTests.class, DevStoreTests.class }) +//@Ignore +/* These test only works on accounts with server-side encryption enabled. */ +public class CloudFileServerEncryptionTests { + + private CloudFileShare share; + private CloudFileDirectory dir; + private CloudFile file; + private boolean requestFound; + + @Before + public void fileEncryptionTestMethodSetup() throws URISyntaxException, StorageException, IOException { + this.share = FileTestHelper.getRandomShareReference(); + this.share.create(); + this.dir = this.share.getRootDirectoryReference().getDirectoryReference("dir"); + this.dir.create(); + this.file = this.share.getRootDirectoryReference().getFileReference("file"); + this.file.uploadText("text"); + } + + @After + public void fileEncryptionTestMethodTearDown() throws StorageException { + this.share.deleteIfExists(); + } + + @Test + public void testFileAttributesEncryption() throws URISyntaxException, StorageException, IOException { + this.file.downloadAttributes(); + assertTrue(this.file.getProperties().isServerEncrypted()); + + CloudFile testFile = this.share.getRootDirectoryReference().getFileReference("file"); + testFile.downloadText(); + assertTrue(testFile.getProperties().isServerEncrypted()); + } + + @Test + public void testDirectoryAttributesEncryption() throws URISyntaxException, StorageException, IOException { + assertFalse(this.dir.getProperties().isServerEncrypted()); + + CloudFileDirectory testDir = this.share.getRootDirectoryReference().getDirectoryReference("dir"); + testDir.downloadAttributes(); + assertTrue(testDir.getProperties().isServerEncrypted()); + } + + @Test + public void testCloudFileUploadEncryption() throws URISyntaxException, StorageException, IOException { + this.requestFound = false; + + OperationContext ctxt = new OperationContext(); + ctxt.getRequestCompletedEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(RequestCompletedEvent eventArg) { + assertTrue(eventArg.getRequestResult().isRequestServiceEncrypted()); + CloudFileServerEncryptionTests.this.requestFound = true; + } + }); + + this.file.uploadText("test", null, null, null, ctxt); + assertTrue(this.requestFound); + + this.requestFound = false; + this.file.uploadProperties(null, null, ctxt); + assertTrue(this.requestFound); + + this.requestFound = false; + this.file.uploadMetadata(null, null, ctxt); + assertTrue(this.requestFound); + } + + @Test + public void testCloudFileDirectoryEncryption() throws URISyntaxException, StorageException, IOException { + this.requestFound = false; + + OperationContext ctxt = new OperationContext(); + ctxt.getRequestCompletedEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(RequestCompletedEvent eventArg) { + assertTrue(eventArg.getRequestResult().isRequestServiceEncrypted()); + CloudFileServerEncryptionTests.this.requestFound = true; + } + }); + + this.dir.uploadMetadata(null, null, ctxt); + assertTrue(this.requestFound); + + this.requestFound = false; + CloudFileDirectory dir2 = this.share.getRootDirectoryReference().getDirectoryReference("dir2"); + dir2.create(null, ctxt); + assertTrue(this.requestFound); + } +} \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudAppendBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudAppendBlob.java index 03cae586d9c68..b54f3be8323c2 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudAppendBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudAppendBlob.java @@ -34,6 +34,7 @@ import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.core.BaseResponse; import com.microsoft.azure.storage.core.ExecutionEngine; import com.microsoft.azure.storage.core.SR; import com.microsoft.azure.storage.core.StorageRequest; @@ -309,7 +310,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); blob.getProperties().setLength(0); return null; } @@ -460,7 +461,7 @@ public Long preProcessResponse(CloudAppendBlob blob, CloudBlobClient client, Ope blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); blob.updateCommittedBlockCountFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return appendOffset; } diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java index 9ab2374340a02..c814ad7699fa4 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java @@ -44,6 +44,7 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageLocation; import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.core.BaseResponse; import com.microsoft.azure.storage.core.ExecutionEngine; import com.microsoft.azure.storage.core.Logger; import com.microsoft.azure.storage.core.NetworkInputStream; @@ -2787,7 +2788,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; @@ -2937,8 +2938,4 @@ protected static String getParentNameFromURI(final StorageUri resourceAddress, f return parentName; } - - protected static boolean isServerRequestEncrypted(HttpURLConnection connection) { - return Constants.TRUE.equals(connection.getHeaderField(Constants.HeaderConstants.SERVER_REQUEST_ENCRYPTED)); - } } \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlockBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlockBlob.java index 301ca433e32a2..f8be120fa7d81 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlockBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlockBlob.java @@ -20,6 +20,7 @@ import javax.crypto.Cipher; import javax.xml.stream.XMLStreamException; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -379,7 +380,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } @@ -915,7 +916,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } @@ -1111,7 +1112,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation return null; } - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java index 53863bed5171a..f728807d009b9 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java @@ -35,6 +35,7 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; import com.microsoft.azure.storage.core.Base64; +import com.microsoft.azure.storage.core.BaseResponse; import com.microsoft.azure.storage.core.ExecutionEngine; import com.microsoft.azure.storage.core.RequestLocationMode; import com.microsoft.azure.storage.core.SR; @@ -510,7 +511,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); blob.getProperties().setLength(length); return null; } @@ -1050,7 +1051,7 @@ public Void preProcessResponse(CloudPageBlob blob, CloudBlobClient client, Opera blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); blob.updateSequenceNumberFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/BaseResponse.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/BaseResponse.java index 555fa0a9ea814..68b75b0d8b91f 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/BaseResponse.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/BaseResponse.java @@ -80,6 +80,16 @@ public static String getRequestId(final HttpURLConnection request) { return request.getHeaderField(Constants.HeaderConstants.REQUEST_ID_HEADER); } + /** + * Gets if the request was encrypted by the server. + * @param request + * The response from the server. + * @return A boolean indicating if the request was encrypted by the server. + */ + public static boolean isServerRequestEncrypted(HttpURLConnection request) { + return Constants.TRUE.equals(request.getHeaderField(Constants.HeaderConstants.SERVER_REQUEST_ENCRYPTED)); + } + /** * Returns all the header/value pairs with the given prefix. * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFile.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFile.java index b9406cbf625f7..75432df9d11e6 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFile.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFile.java @@ -50,6 +50,7 @@ import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.core.Base64; +import com.microsoft.azure.storage.core.BaseResponse; import com.microsoft.azure.storage.core.ExecutionEngine; import com.microsoft.azure.storage.core.Logger; import com.microsoft.azure.storage.core.NetworkInputStream; @@ -667,6 +668,7 @@ public Void preProcessResponse(CloudFile file, CloudFileClient client, Operation } file.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } @@ -2284,6 +2286,7 @@ public Void preProcessResponse(CloudFile file, CloudFileClient client, Operation } file.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; @@ -2376,6 +2379,7 @@ public Void preProcessResponse(CloudFile file, CloudFileClient client, Operation } file.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; @@ -2462,6 +2466,7 @@ public Void preProcessResponse(CloudFile file, CloudFileClient client, Operation } file.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; @@ -2544,6 +2549,7 @@ public Void preProcessResponse(CloudFile file, CloudFileClient client, Operation file.getProperties().setLength(size); file.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFileDirectory.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFileDirectory.java index b2d063e6cf6df..0caf8fd5ba1bc 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFileDirectory.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFileDirectory.java @@ -34,6 +34,7 @@ import com.microsoft.azure.storage.StorageErrorCodeStrings; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.core.BaseResponse; import com.microsoft.azure.storage.core.ExecutionEngine; import com.microsoft.azure.storage.core.LazySegmentedIterable; import com.microsoft.azure.storage.core.ListResponse; @@ -238,6 +239,7 @@ public Void preProcessResponse(CloudFileDirectory directory, CloudFileClient cli final FileDirectoryAttributes attributes = FileResponse .getFileDirectoryAttributes(this.getConnection(), client.isUsePathStyleUris()); directory.setProperties(attributes.getProperties()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; @@ -629,6 +631,7 @@ public Void preProcessResponse(CloudFileDirectory directory, CloudFileClient cli } directory.updatePropertiesFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(BaseResponse.isServerRequestEncrypted(this.getConnection())); return null; } }; diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileDirectoryProperties.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileDirectoryProperties.java index c6dcfd3cc8038..525f2d22ca1fa 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileDirectoryProperties.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileDirectoryProperties.java @@ -33,6 +33,11 @@ public final class FileDirectoryProperties { */ private Date lastModified; + /** + * Represents the directory's server-side encryption status. + */ + private boolean serverEncrypted; + /** * Gets the ETag value of the directory. *

@@ -58,6 +63,15 @@ public Date getLastModified() { return this.lastModified; } + /** + * Gets the directory's server-side encryption status. + * + * @return A boolean which specifies the directory's encryption status. + */ + public boolean isServerEncrypted() { + return serverEncrypted; + } + /** * Sets the ETag value on the directory. * @@ -68,6 +82,16 @@ protected void setEtag(final String etag) { this.etag = etag; } + /** + * Sets the directory's server-side encryption status. + * + * @param serverEncrypted + * A boolean which specifies the encryption status to set. + */ + protected void setServerEncrypted(boolean serverEncrypted) { + this.serverEncrypted = serverEncrypted; + } + /** * Sets the last modified time on the directory. * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileProperties.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileProperties.java index f0ded017f865d..9194fba375bac 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileProperties.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileProperties.java @@ -76,6 +76,11 @@ public final class FileProperties { */ private Date lastModified; + /** + * Represents the file's server-side encryption status. + */ + private boolean serverEncrypted; + /** * Creates an instance of the FileProperties class. */ @@ -100,6 +105,7 @@ public FileProperties(final FileProperties other) { this.etag = other.etag; this.length = other.length; this.lastModified = other.lastModified; + this.serverEncrypted = other.serverEncrypted; } /** @@ -203,6 +209,15 @@ public long getLength() { return this.length; } + /** + * Gets the file's server-side encryption status. + * + * @return A boolean which specifies the file's encryption status. + */ + public boolean isServerEncrypted() { + return serverEncrypted; + } + /** * Sets the cache control value for the file. * @@ -262,7 +277,17 @@ public void setContentMD5(final String contentMD5) { public void setContentType(final String contentType) { this.contentType = contentType; } - + + /** + * Sets the file's server-side encryption status. + * + * @param serverEncrypted + * A boolean which specifies the encryption status to set. + */ + protected void setServerEncrypted(boolean serverEncrypted) { + this.serverEncrypted = serverEncrypted; + } + /** * Sets the copy state value for the file. * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileResponse.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileResponse.java index 14d7bcfcd98c6..fe9a0090701fa 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileResponse.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileResponse.java @@ -125,6 +125,8 @@ public static FileDirectoryAttributes getFileDirectoryAttributes(final HttpURLCo directoryProperties.setEtag(BaseResponse.getEtag(request)); directoryProperties.setLastModified(new Date(request.getLastModified())); directoryAttributes.setMetadata(getMetadata(request)); + directoryProperties.setServerEncrypted( + Constants.TRUE.equals(request.getHeaderField(Constants.HeaderConstants.SERVER_ENCRYPTED))); return directoryAttributes; } @@ -161,6 +163,8 @@ public static FileAttributes getFileAttributes(final HttpURLConnection request, properties.setContentType(request.getHeaderField(Constants.HeaderConstants.CONTENT_TYPE)); properties.setEtag(BaseResponse.getEtag(request)); properties.setCopyState(FileResponse.getCopyState(request)); + properties.setServerEncrypted( + Constants.TRUE.equals(request.getHeaderField(Constants.HeaderConstants.SERVER_ENCRYPTED))); final Calendar lastModifiedCalendar = Calendar.getInstance(Utility.LOCALE_US); lastModifiedCalendar.setTimeZone(Utility.UTC_ZONE); From 98b2d83fc50b28833c0bffa34bea5e8bd7e55ad7 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Tue, 27 Jun 2017 16:22:42 -0700 Subject: [PATCH 2/7] Add error receiving response event --- ChangeLog.txt | 3 + .../azure/storage/EventFiringTests.java | 103 +++++++++++++++++- .../storage/ErrorReceivingResponseEvent.java | 38 +++++++ .../azure/storage/OperationContext.java | 68 +++++++++++- .../azure/storage/core/ExecutionEngine.java | 100 +++++++++-------- .../azure/storage/core/LogConstants.java | 1 + 6 files changed, 262 insertions(+), 51 deletions(-) create mode 100644 microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java diff --git a/ChangeLog.txt b/ChangeLog.txt index 920ca54f303a4..06394b55e4607 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,3 +1,6 @@ +2017.XX.XX Version X.X.X + * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire. + 2017.06.21 Version 5.3.1 * Fixed a bug in specific upload case for block blobs. This only affects uploads greater than the max put blob threshold, that have increased the streamWriteSizeInBytes beyond the 4 MB and storeBlobContentMD5 has been disabled. * In some cases in the above mentioned upload path, fixed a bug where StorageExceptions were being thrown instead of IOExceptions. diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java index 4ea007be63c96..6ee2236d86e86 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java @@ -14,18 +14,20 @@ */ package com.microsoft.azure.storage; -import com.microsoft.azure.storage.blob.BlobRequestOptions; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.*; import com.microsoft.azure.storage.core.SR; import com.microsoft.azure.storage.TestRunners.CloudTests; import com.microsoft.azure.storage.TestRunners.DevFabricTests; import com.microsoft.azure.storage.TestRunners.DevStoreTests; +import org.apache.http.protocol.HTTP; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.net.HttpURLConnection; +import java.net.SocketException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -111,6 +113,22 @@ public void eventOccurred(ResponseReceivedEvent eventArg) { } }); + eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + fail("This event should not trigger"); + } + }); + + OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + fail("This event should not trigger"); + } + }); + assertEquals(0, callList.size()); assertEquals(0, globalCallList.size()); @@ -138,6 +156,85 @@ public void eventOccurred(ResponseReceivedEvent eventArg) { assertEquals(2, globalCallList.size()); } + @Test + public void testErrorReceivingResponseEvent() throws URISyntaxException, StorageException { + final ArrayList callList = new ArrayList(); + final ArrayList globalCallList = new ArrayList(); + + OperationContext eventContext = new OperationContext(); + BlobRequestOptions options = new BlobRequestOptions(); + options.setRetryPolicyFactory(new RetryNoRetry()); + + // setting the sending request event handler to trigger an exception. + // this is a retryable exception + eventContext.getSendingRequestEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(SendingRequestEvent eventArg) { + HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject(); + connection.setFixedLengthStreamingMode(0); + } + }); + + eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult()); + callList.add(true); + } + }); + + OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult()); + globalCallList.add(true); + } + }); + + CloudBlobClient blobClient = TestHelper.createCloudBlobClient(); + CloudBlobContainer container = blobClient.getContainerReference("container1"); + container.createIfNotExists(); + + try { + CloudBlockBlob blob1 = container.getBlockBlobReference("blob1"); + try { + String blockID = String.format("%08d", 1); + blob1.uploadBlock(blockID, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + // make sure both the local and globab context update + assertEquals(1, callList.size()); + assertEquals(1, globalCallList.size()); + + // make sure only global updates by replacing the local with a no-op event + eventContext + .setErrorReceivingResponseEventHandler(new StorageEventMultiCaster>()); + try { + String blockID2 = String.format("%08d", 2); + blob1.uploadBlock(blockID2, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + assertEquals(1, callList.size()); + assertEquals(2, globalCallList.size()); + + // make sure global does not update by replacing the global with a no-op + OperationContext + .setGlobalErrorReceivingResponseEventHandler(new StorageEventMultiCaster>()); + + // make sure neither update + try { + String blockID3 = String.format("%08d", 3); + blob1.uploadBlock(blockID3, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + assertEquals(1, callList.size()); + assertEquals(2, globalCallList.size()); + } + finally { + container.deleteIfExists(); + } + } + @Test public void testRequestCompletedEvents() throws URISyntaxException, StorageException { final ArrayList callList = new ArrayList(); diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java new file mode 100644 index 0000000000000..84faacaa55765 --- /dev/null +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java @@ -0,0 +1,38 @@ +/** + * Copyright Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.microsoft.azure.storage; + +/** + * Represents an event that is fired when a network error occurs before the HTTP response status and headers are received. + */ +public final class ErrorReceivingResponseEvent extends BaseEvent { + + /** + * Creates an instance of the BaseEvent class that is fired when a network error occurs before the HTTP response status and headers are received. + * + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * @param connectionObject + * Represents a connection object. Currently only java.net.HttpURLConnection is supported as + * a connection object. + * @param requestResult + * A {@link RequestResult} object that represents the current request result. + */ + public ErrorReceivingResponseEvent(OperationContext opContext, Object connectionObject, RequestResult requestResult) { + super(opContext, connectionObject, requestResult); + } +} \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java index a59534d9f8dc5..0564fd01cee9a 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java @@ -83,7 +83,8 @@ public final class OperationContext { private HashMap userHeaders; /** - * Represents an event that is triggered before sending a request. + * Represents an event that is triggered before sending a + * request. * * @see StorageEvent * @see StorageEventMultiCaster @@ -92,15 +93,23 @@ public final class OperationContext { private static StorageEventMultiCaster> globalSendingRequestEventHandler = new StorageEventMultiCaster>(); /** - * Represents an event that is triggered when a response is received from the storage service while processing a - * request. - * + * Represents an event that is triggered when a response is received from the storage service while processing a request + * * @see StorageEvent * @see StorageEventMultiCaster * @see ResponseReceivedEvent */ private static StorageEventMultiCaster> globalResponseReceivedEventHandler = new StorageEventMultiCaster>(); + /** + * Represents an event that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @see StorageEvent + * @see StorageEventMultiCaster + * @see ErrorReceivingResponseEvent + */ + private static StorageEventMultiCaster> globalErrorReceivingResponseEventHandler = new StorageEventMultiCaster>(); + /** * Represents an event that is triggered when a response received from the service is fully processed. * @@ -138,6 +147,15 @@ public final class OperationContext { */ private StorageEventMultiCaster> responseReceivedEventHandler = new StorageEventMultiCaster>(); + /** + * Represents an event that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @see StorageEvent + * @see StorageEventMultiCaster + * @see ErrorReceivingResponseEvent + */ + private StorageEventMultiCaster> errorReceivingResponseEventHandler = new StorageEventMultiCaster>(); + /** * Represents an event that is triggered when a response received from the service is fully processed. * @@ -285,6 +303,16 @@ public static StorageEventMultiCasterglobabErrorReceivingResponseEventHandler. + */ + public static StorageEventMultiCaster> getGlobalErrorReceivingResponseEventHandler() { + return OperationContext.globalErrorReceivingResponseEventHandler; + } + /** * Gets a global event multi-caster that is triggered when a request is completed. It allows event listeners to be * dynamically added and removed. @@ -325,6 +353,16 @@ public StorageEventMultiCastererrorReceivingResponseEventHandler. + */ + public StorageEventMultiCaster> getErrorReceivingResponseEventHandler() { + return this.errorReceivingResponseEventHandler; + } + /** * Gets an event multi-caster that is triggered when a request is completed. It allows event listeners to be * dynamically added and removed. @@ -450,6 +488,17 @@ public static void setGlobalResponseReceivedEventHandler( OperationContext.globalResponseReceivedEventHandler = globalResponseReceivedEventHandler; } + /** + * Sets a global event multi-caster that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @param globalErrorReceivingResponseEventHandler + * The {@link StorageEventMultiCaster} object to set for the globalErrorReceivingResponseEventHandler. + */ + public static void setGlobalErrorReceivingResponseEventHandler( + final StorageEventMultiCaster> globalErrorReceivingResponseEventHandler) { + OperationContext.globalErrorReceivingResponseEventHandler = globalErrorReceivingResponseEventHandler; + } + /** * Sets a global event multi-caster that is triggered when a request is completed. * @@ -494,6 +543,17 @@ public void setResponseReceivedEventHandler( this.responseReceivedEventHandler = responseReceivedEventHandler; } + /** + * Sets an event multi-caster that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @param errorReceivingResponseEventHandler + * The {@link StorageEventMultiCaster} object to set for the errorReceivingResponseEventHandler. + */ + public void setErrorReceivingResponseEventHandler( + final StorageEventMultiCaster> errorReceivingResponseEventHandler) { + this.errorReceivingResponseEventHandler = errorReceivingResponseEventHandler; + } + /** * Sets an event multi-caster that is triggered when a request is completed. * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java index 919f7811d7942..06cf8c8e03baf 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java @@ -22,22 +22,7 @@ import java.util.Map.Entry; import java.util.concurrent.TimeoutException; -import com.microsoft.azure.storage.Constants; -import com.microsoft.azure.storage.LocationMode; -import com.microsoft.azure.storage.OperationContext; -import com.microsoft.azure.storage.RequestCompletedEvent; -import com.microsoft.azure.storage.RequestResult; -import com.microsoft.azure.storage.ResponseReceivedEvent; -import com.microsoft.azure.storage.RetryContext; -import com.microsoft.azure.storage.RetryInfo; -import com.microsoft.azure.storage.RetryNoRetry; -import com.microsoft.azure.storage.RetryPolicy; -import com.microsoft.azure.storage.RetryPolicyFactory; -import com.microsoft.azure.storage.RetryingEvent; -import com.microsoft.azure.storage.SendingRequestEvent; -import com.microsoft.azure.storage.StorageErrorCodeStrings; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.StorageLocation; +import com.microsoft.azure.storage.*; /** * RESERVED FOR INTERNAL USE. A class that handles execution of StorageOperations and enforces retry policies. @@ -98,41 +83,55 @@ public static RESULT_TYPE executeWithRet request.getRequestProperty(Constants.HeaderConstants.DATE)); // 5. Potentially upload data - if (task.getSendStream() != null) { - Logger.info(opContext, LogConstants.UPLOAD); - final StreamMd5AndLength descriptor = Utility.writeToOutputStream(task.getSendStream(), - request.getOutputStream(), task.getLength(), false /* rewindStream */, - false /* calculate MD5 */, opContext, task.getRequestOptions()); - - task.validateStreamWrite(descriptor); - Logger.info(opContext, LogConstants.UPLOADDONE); - } + boolean responseReceivedEventTriggered = false; + try { + if (task.getSendStream() != null) { + Logger.info(opContext, LogConstants.UPLOAD); + final StreamMd5AndLength descriptor = Utility.writeToOutputStream(task.getSendStream(), + request.getOutputStream(), task.getLength(), false /* rewindStream */, + false /* calculate MD5 */, opContext, task.getRequestOptions()); + + task.validateStreamWrite(descriptor); + Logger.info(opContext, LogConstants.UPLOADDONE); + } + + Utility.logHttpRequest(request, opContext); - Utility.logHttpRequest(request, opContext); + // 6. Process the request - Get response + RequestResult currResult = task.getResult(); + currResult.setStartDate(new Date()); - // 6. Process the request - Get response - RequestResult currResult = task.getResult(); - currResult.setStartDate(new Date()); + Logger.info(opContext, LogConstants.GET_RESPONSE); - Logger.info(opContext, LogConstants.GET_RESPONSE); + currResult.setStatusCode(request.getResponseCode()); + currResult.setStatusMessage(request.getResponseMessage()); + currResult.setStopDate(new Date()); - currResult.setStatusCode(request.getResponseCode()); - currResult.setStatusMessage(request.getResponseMessage()); - currResult.setStopDate(new Date()); + currResult.setServiceRequestID(BaseResponse.getRequestId(request)); + currResult.setEtag(BaseResponse.getEtag(request)); + currResult.setRequestDate(BaseResponse.getDate(request)); + currResult.setContentMD5(BaseResponse.getContentMD5(request)); - currResult.setServiceRequestID(BaseResponse.getRequestId(request)); - currResult.setEtag(BaseResponse.getEtag(request)); - currResult.setRequestDate(BaseResponse.getDate(request)); - currResult.setContentMD5(BaseResponse.getContentMD5(request)); + // 7. Fire ResponseReceived Event + responseReceivedEventTriggered = true; + ExecutionEngine.fireResponseReceivedEvent(opContext, request, task.getResult()); - // 7. Fire ResponseReceived Event - ExecutionEngine.fireResponseReceivedEvent(opContext, request, task.getResult()); + Logger.info(opContext, LogConstants.RESPONSE_RECEIVED, currResult.getStatusCode(), + currResult.getServiceRequestID(), currResult.getContentMD5(), currResult.getEtag(), + currResult.getRequestDate()); - Logger.info(opContext, LogConstants.RESPONSE_RECEIVED, currResult.getStatusCode(), - currResult.getServiceRequestID(), currResult.getContentMD5(), currResult.getEtag(), - currResult.getRequestDate()); - - Utility.logHttpResponse(request, opContext); + Utility.logHttpResponse(request, opContext); + } + finally { + Logger.info(opContext, LogConstants.ERROR_RECEIVING_RESPONSE); + if (!responseReceivedEventTriggered) { + if (task.getResult().getStartDate() == null) { + task.getResult().setStartDate(new Date()); + } + + ExecutionEngine.fireErrorReceivingResponseEvent(opContext, request, task.getResult()); + } + } // 8. Pre-process response to check if there was an exception. Do Response parsing (headers etc). Logger.info(opContext, LogConstants.PRE_PROCESS); @@ -376,6 +375,19 @@ private static void fireResponseReceivedEvent(OperationContext opContext, HttpUR } } + /** + * Fires events representing that an error occurred when receiving the response. + */ + private static void fireErrorReceivingResponseEvent(OperationContext opContext, HttpURLConnection request, + RequestResult result) { + if (opContext.getErrorReceivingResponseEventHandler().hasListeners() + || OperationContext.getGlobalErrorReceivingResponseEventHandler().hasListeners()) { + ErrorReceivingResponseEvent event = new ErrorReceivingResponseEvent(opContext, request, result); + opContext.getErrorReceivingResponseEventHandler().fireEvent(event); + OperationContext.getGlobalErrorReceivingResponseEventHandler().fireEvent(event); + } + } + /** * Fires events representing that a response received from the service is fully processed. */ diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java index 55fd5be15c245..a29cc3c4430a5 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java @@ -22,6 +22,7 @@ public class LogConstants { public static final String COMPLETE = "Operation completed."; public static final String DO_NOT_RETRY_POLICY = "Retry policy did not allow for a retry. Failing. Error Message = '%s'."; public static final String DO_NOT_RETRY_TIMEOUT = "Operation cannot be retried because maximum execution timeout has been reached. Failing. Inner error Message = '%s'."; + public static final String ERROR_RECEIVING_RESPONSE = "A network error occurred before the HTTP response status and headers were received."; public static final String GET_RESPONSE = "Waiting for response."; public static final String INIT_LOCATION = "Starting operation with location '%s' per location mode '%s'."; public static final String NEXT_LOCATION = "The next location has been set to '%s', per location mode '%s'."; From ecde918aecc770f1fa2b73d22b88f82073cd1e33 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Fri, 7 Jul 2017 10:09:06 -0700 Subject: [PATCH 3/7] Eight TB Test --- .../storage/blob/CloudPageBlobTests.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java index 9dd0e6b265f0e..3282a1e6a9c15 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java @@ -1200,4 +1200,36 @@ else if (overload == 2) { assertNotNull(copy.properties.getCopyState().getCopyDestinationSnapshotID()); assertNotNull(copy.getCopyState().getCompletionTime()); } + + @Test + public void testEightTBBlob() throws StorageException, URISyntaxException, IOException { + CloudPageBlob blob = this.container.getPageBlobReference("blob1"); + CloudPageBlob blob2 = this.container.getPageBlobReference("blob1"); + + long eightTb = 8L * 1024L * 1024L * 1024L * 1024L; + blob.create(eightTb); + assertEquals(eightTb, blob.getProperties().getLength()); + + blob2.downloadAttributes(); + assertEquals(eightTb, blob2.getProperties().getLength()); + + for (ListBlobItem listBlob : this.container.listBlobs()) { + CloudPageBlob listPageBlob = (CloudPageBlob)listBlob; + assertEquals(eightTb, listPageBlob.getProperties().getLength()); + } + + CloudPageBlob blob3 = this.container.getPageBlobReference("blob3"); + blob3.create(1024); + blob3.resize(eightTb); + + final Random randGenerator = new Random(); + final byte[] buffer = new byte[1024]; + randGenerator.nextBytes(buffer); + blob.uploadPages(new ByteArrayInputStream(buffer), eightTb - 512L, 512L); + + ArrayList ranges = blob.downloadPageRanges(); + assertEquals(1, ranges.size()); + assertEquals(eightTb - 512L, ranges.get(0).getStartOffset()); + assertEquals(eightTb - 1L, ranges.get(0).getEndOffset()); + } } From 44be0109c78cc9f842916a8f393b91830f2774c5 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Tue, 11 Jul 2017 10:35:45 -0700 Subject: [PATCH 4/7] Premium Page Blob Tiers --- ChangeLog.txt | 1 + .../res/TestConfigurations.xml | 1 + .../microsoft/azure/storage/TestHelper.java | 112 +++++- .../microsoft/azure/storage/TestRunners.java | 11 +- .../azure/storage/blob/BlobTestHelper.java | 8 + .../storage/blob/CloudPageBlobTests.java | 176 +++++++++- .../microsoft/azure/storage/Constants.java | 9 +- .../azure/storage/blob/BlobConstants.java | 13 +- .../azure/storage/blob/BlobListHandler.java | 5 + .../azure/storage/blob/BlobProperties.java | 48 ++- .../azure/storage/blob/BlobRequest.java | 95 ++++- .../azure/storage/blob/BlobResponse.java | 20 ++ .../azure/storage/blob/CloudBlob.java | 49 ++- .../azure/storage/blob/CloudPageBlob.java | 327 ++++++++++++++++-- .../storage/blob/PremiumPageBlobTier.java | 112 ++++++ 15 files changed, 946 insertions(+), 41 deletions(-) create mode 100644 microsoft-azure-storage/src/com/microsoft/azure/storage/blob/PremiumPageBlobTier.java diff --git a/ChangeLog.txt b/ChangeLog.txt index 06394b55e4607..54c0ac5b3f71a 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,5 +1,6 @@ 2017.XX.XX Version X.X.X * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire. + * For Premium Accounts only, added support for getting and setting the tier on a page blob. The tier can also be set when creating or copying from an existing page blob. 2017.06.21 Version 5.3.1 * Fixed a bug in specific upload case for block blobs. This only affects uploads greater than the max put blob threshold, that have increased the streamWriteSizeInBytes beyond the 4 MB and storeBlobContentMD5 has been disabled. diff --git a/microsoft-azure-storage-test/res/TestConfigurations.xml b/microsoft-azure-storage-test/res/TestConfigurations.xml index bb3250dd43dc2..edce1b7f2e1d3 100644 --- a/microsoft-azure-storage-test/res/TestConfigurations.xml +++ b/microsoft-azure-storage-test/res/TestConfigurations.xml @@ -1,5 +1,6 @@ ProductionTenant +ProductionTenant DevStore diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java index c6ae73860536b..490b8df5baeb1 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java @@ -15,6 +15,7 @@ package com.microsoft.azure.storage; import static org.junit.Assert.*; +import static org.junit.Assume.assumeNotNull; import java.io.ByteArrayInputStream; import java.io.File; @@ -39,6 +40,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.junit.AssumptionViolatedException; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Node; @@ -58,6 +60,9 @@ public class TestHelper { private static Tenant tenant; private static StorageCredentialsAccountAndKey credentials; private static CloudStorageAccount account; + private static Tenant premiumBlobTenant; + private static StorageCredentialsAccountAndKey premiumBlobCredentials; + private static CloudStorageAccount premiumBlobAccount; private final static boolean enableFiddler = true; private final static boolean requireSecondaryEndpoint = false; @@ -67,6 +72,11 @@ public static CloudBlobClient createCloudBlobClient() throws StorageException { return client; } + public static CloudBlobClient createPremiumCloudBlobClient() throws StorageException { + CloudBlobClient client = getPremiumBlobAccount().createCloudBlobClient(); + return client; + } + public static CloudBlobClient createCloudBlobClient(SharedAccessAccountPolicy policy, boolean useHttps) throws StorageException, InvalidKeyException, URISyntaxException { @@ -328,12 +338,12 @@ private static CloudStorageAccount getAccount() throws StorageException { account = CloudStorageAccount.parse(cloudAccount); } else if (accountConfig != null) { - tenant = readTestConfigsFromXml(new File(accountConfig)); + readTestConfigsFromXml(new File(accountConfig), false); setAccountAndCredentials(); } else { URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml"); - tenant = readTestConfigsFromXml(new File(localTestConfig.getPath())); + readTestConfigsFromXml(new File(localTestConfig.getPath()), false); setAccountAndCredentials(); } } @@ -344,6 +354,47 @@ else if (accountConfig != null) { return account; } + private static CloudStorageAccount getPremiumBlobAccount() throws StorageException { + // Only do this the first time TestBase is called as storage account is static + if (premiumBlobAccount == null) { + //enable fiddler + if (enableFiddler) + enableFiddler(); + + // try to get the environment variable with the test configuration file path + String accountConfig; + try { + accountConfig = System.getenv("storageTestConfiguration"); + } + catch (SecurityException e) { + accountConfig = null; + } + + // if storageConnection is set, use that as an account string + // if storageTestConfiguration is set, use that as a path to the configurations file + // if neither are set, use the local configurations file at TestConfigurations.xml + try { + if (accountConfig != null) { + readTestConfigsFromXml(new File(accountConfig), true); + setAccountAndCredentials(); + } + else { + URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml"); + readTestConfigsFromXml(new File(localTestConfig.getPath()), true); + setAccountAndCredentials(); + } + } + catch (AssumptionViolatedException e) { + throw e; + } + catch (Exception e) { + throw StorageException.translateClientException(e); + } + } + + return premiumBlobAccount; + } + private static void setAccountAndCredentials() { if (requireSecondaryEndpoint) tenant.assertSecondaryEndpoint(); @@ -353,9 +404,17 @@ private static void setAccountAndCredentials() { tenant.getQueueServiceSecondaryEndpoint()), new StorageUri(tenant.getTableServiceEndpoint(), tenant.getTableServiceSecondaryEndpoint()), new StorageUri(tenant.getFileServiceEndpoint(), tenant.getFileServiceSecondaryEndpoint())); + + if (premiumBlobTenant != null) { + premiumBlobCredentials = new StorageCredentialsAccountAndKey(premiumBlobTenant.getAccountName(), premiumBlobTenant.getAccountKey()); + premiumBlobAccount = new CloudStorageAccount(premiumBlobCredentials, new StorageUri(premiumBlobTenant.getBlobServiceEndpoint(), premiumBlobTenant.getBlobServiceSecondaryEndpoint()), + null, + null, + null); + } } - private static Tenant readTestConfigsFromXml(File testConfigurations) throws ParserConfigurationException, + private static void readTestConfigsFromXml(File testConfigurations, boolean premiumBlob) throws ParserConfigurationException, SAXException, IOException, DOMException, URISyntaxException { DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); @@ -372,7 +431,14 @@ private static Tenant readTestConfigsFromXml(File testConfigurations) throws Par throw new IllegalArgumentException("No TargetTestTenant specified."); } - Tenant tenant = null; + Node premiumBlobTenantNode = testConfigs.getElementsByTagName("TargetPremiumBlobTenant").item(0); + String premiumBlobTenantName = null; + if (premiumBlobTenantNode != null) { + premiumBlobTenantName = premiumBlobTenantNode.getTextContent(); + } + + tenant = null; + premiumBlobTenant = null; final NodeList tenantNodes = testConfigs.getElementsByTagName("TenantName"); for (int i = 0; i < tenantNodes.getLength(); i++) { if (tenantNodes.item(i).getTextContent().equals(targetTenant)) { @@ -436,18 +502,50 @@ else if (name.equals("TableHttpsPortOverride")) { else if (name.equals("FileHttpsPortOverride")) { tenant.setFileHttpsPortOverride(Integer.parseInt(node.getTextContent())); } - else { + else if (!premiumBlob){ throw new IllegalArgumentException(String.format( "Invalid child of TenantConfiguration with name: %s", name)); } } } } + + if (tenantNodes.item(i).getTextContent().equals(premiumBlobTenantName)) { + premiumBlobTenant = new Tenant(); + Node parent = tenantNodes.item(i).getParentNode(); + final NodeList childNodes = parent.getChildNodes(); + for (int j = 0; j < childNodes.getLength(); j++) { + final Node node = childNodes.item(j); + + if (node.getNodeType() != Node.ELEMENT_NODE) { + // do nothing + } else { + final String name = node.getNodeName(); + + if (name.equals("TenantName")) { + premiumBlobTenant.setTenantName(node.getTextContent()); + } else if (name.equals("TenantType")) { + // do nothing, we don't track this field + } else if (name.equals("AccountName")) { + premiumBlobTenant.setAccountName(node.getTextContent()); + } else if (name.equals("AccountKey")) { + premiumBlobTenant.setAccountKey(node.getTextContent()); + } else if (name.equals("BlobServiceEndpoint")) { + premiumBlobTenant.setBlobServiceEndpoint(new URI(node.getTextContent())); + } else if (name.equals("BlobServiceSecondaryEndpoint")) { + premiumBlobTenant.setBlobServiceSecondaryEndpoint(new URI(node.getTextContent())); + } + } + } + } } - if (tenant == null) { + if (tenant == null && !premiumBlob) { throw new IllegalArgumentException("TargetTestTenant specified did not exist in TenantConfigurations."); } - return tenant; + + if (premiumBlobTenant == null && premiumBlob) { + assumeNotNull(premiumBlobTenant); + } } } diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestRunners.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestRunners.java index 81e98d61af39c..831f4c8d093d8 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestRunners.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestRunners.java @@ -105,6 +105,9 @@ public interface CloudTests { public interface DevFabricTests { } + public interface PremiumBlobTests { + } + // Test suites @RunWith(Suite.class) @SuiteClasses({ AccountSasTests.class, EventFiringTests.class, GenericTests.class, LoggerTests.class, @@ -116,7 +119,8 @@ public static class CoreTestSuite { @RunWith(Suite.class) @SuiteClasses({ BlobOutputStreamTests.class, CloudBlobClientTests.class, CloudBlobContainerTests.class, CloudBlobDirectoryTests.class, CloudAppendBlobTests.class, CloudBlockBlobTests.class, CloudPageBlobTests.class, - CloudBlobClientEncryptionTests.class, CloudBlobServerEncryptionTests.class, LeaseTests.class, SasTests.class }) + CloudBlobClientEncryptionTests.class, CloudBlobServerEncryptionTests.class, LeaseTests.class, SasTests.class, + PremiumBlobTests.class }) public static class BlobTestSuite { } @@ -177,4 +181,9 @@ public static class DevFabricNoSecondarySuite { @SuiteClasses(AllTestSuite.class) public static class FastTestSuite { } + + @RunWith(Categories.class) + @IncludeCategory(PremiumBlobTests.class) + public static class PremiumBlobTestSuite { + } } \ No newline at end of file diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java index 480aa25771ee0..9daf49ef0d07b 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java @@ -50,6 +50,14 @@ public static CloudBlobContainer getRandomContainerReference() throws URISyntaxE return container; } + public static CloudBlobContainer getRandomPremiumBlobContainerReference() throws URISyntaxException, StorageException { + String containerName = generateRandomContainerName(); + CloudBlobClient bClient = TestHelper.createPremiumCloudBlobClient(); + CloudBlobContainer container = bClient.getContainerReference(containerName); + + return container; + } + public static String generateRandomBlobNameWithPrefix(String prefix) { if (prefix == null) { prefix = ""; diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java index 3282a1e6a9c15..c8657699d40a5 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java @@ -14,8 +14,6 @@ */ package com.microsoft.azure.storage.blob; -import junit.framework.Assert; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -59,6 +57,7 @@ import com.microsoft.azure.storage.TestRunners.CloudTests; import com.microsoft.azure.storage.TestRunners.DevFabricTests; import com.microsoft.azure.storage.TestRunners.DevStoreTests; +import com.microsoft.azure.storage.TestRunners.PremiumBlobTests; import com.microsoft.azure.storage.core.SR; import com.microsoft.azure.storage.core.UriQueryBuilder; import com.microsoft.azure.storage.core.Utility; @@ -1232,4 +1231,177 @@ public void testEightTBBlob() throws StorageException, URISyntaxException, IOExc assertEquals(eightTb - 512L, ranges.get(0).getStartOffset()); assertEquals(eightTb - 1L, ranges.get(0).getEndOffset()); } + + @Test + @Category(PremiumBlobTests.class) + public void testCloudPageBlobSetPremiumBlobTierOnCreate() throws URISyntaxException, StorageException, IOException { + CloudBlobContainer container = BlobTestHelper.getRandomPremiumBlobContainerReference(); + try { + container.create(); + String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testblob"); + + // Test create API + CloudPageBlob blob = container.getPageBlobReference(blobName); + assertNull(blob.getProperties().getInferredBlobTier()); + blob.create(1024, PremiumPageBlobTier.P4, null, null, null); + assertEquals(PremiumPageBlobTier.P4, blob.getProperties().getPremiumPageBlobTier()); + assertFalse(blob.getProperties().getInferredBlobTier()); + + CloudPageBlob blob2 = container.getPageBlobReference(blobName); + blob2.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P4, blob2.getProperties().getPremiumPageBlobTier()); + assertNull(blob2.getProperties().getInferredBlobTier()); + + // Test upload from byte array API + byte[] buffer = BlobTestHelper.getRandomBuffer(1024); + CloudPageBlob blob3 = container.getPageBlobReference("blob3"); + blob3.uploadFromByteArray(buffer, 0, 1024, PremiumPageBlobTier.P6, null, null, null); + assertEquals(PremiumPageBlobTier.P6, blob3.getProperties().getPremiumPageBlobTier()); + assertFalse(blob3.getProperties().getInferredBlobTier()); + + CloudPageBlob blob3Ref = container.getPageBlobReference("blob3"); + blob3Ref.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P6, blob3Ref.getProperties().getPremiumPageBlobTier()); + assertNull(blob3Ref.getProperties().getInferredBlobTier()); + + // Test upload from stream API + ByteArrayInputStream srcStream = new ByteArrayInputStream(buffer); + CloudPageBlob blob4 = container.getPageBlobReference("blob4"); + blob4.upload(srcStream, 1024, PremiumPageBlobTier.P10, null, null, null); + assertEquals(PremiumPageBlobTier.P10, blob4.getProperties().getPremiumPageBlobTier()); + assertFalse(blob4.getProperties().getInferredBlobTier()); + + CloudPageBlob blob4Ref = container.getPageBlobReference("blob4"); + blob4Ref.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P10, blob4Ref.getProperties().getPremiumPageBlobTier()); + assertNull(blob4Ref.getProperties().getInferredBlobTier()); + + // Test upload from file API + File sourceFile = File.createTempFile("sourceFile", ".tmp"); + File destinationFile = new File(sourceFile.getParentFile(), + "destinationFile.tmp"); + FileOutputStream fos = new FileOutputStream(sourceFile); + fos.write(buffer); + fos.close(); + CloudPageBlob blob5 = container.getPageBlobReference("blob5"); + blob5.uploadFromFile(sourceFile.getAbsolutePath(), PremiumPageBlobTier.P20, null, null, null); + assertEquals(PremiumPageBlobTier.P20, blob5.getProperties().getPremiumPageBlobTier()); + assertFalse(blob5.getProperties().getInferredBlobTier()); + + CloudPageBlob blob5Ref = container.getPageBlobReference("blob5"); + blob5Ref.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P20, blob5Ref.getProperties().getPremiumPageBlobTier()); + assertNull(blob5Ref.getProperties().getInferredBlobTier()); + } + finally { + container.deleteIfExists(); + } + } + + @Test + @Category(PremiumBlobTests.class) + public void testCloudPageBlobSetBlobTier() throws URISyntaxException, StorageException { + CloudBlobContainer container = BlobTestHelper.getRandomPremiumBlobContainerReference(); + try { + container.create(); + String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testblob"); + CloudPageBlob blob = container.getPageBlobReference(blobName); + blob.create(1024); + assertNull(blob.getProperties().getInferredBlobTier()); + blob.downloadAttributes(); + assertTrue(blob.getProperties().getInferredBlobTier()); + assertEquals(PremiumPageBlobTier.P10, blob.getProperties().getPremiumPageBlobTier()); + + blob.uploadPremiumPageBlobTier(PremiumPageBlobTier.P40); + assertEquals(PremiumPageBlobTier.P40, blob.properties.getPremiumPageBlobTier()); + assertFalse(blob.getProperties().getInferredBlobTier()); + + CloudPageBlob blob2 = container.getPageBlobReference(blobName); + blob2.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P40, blob2.properties.getPremiumPageBlobTier()); + assertNull(blob2.getProperties().getInferredBlobTier()); + + boolean pageBlobWithTierFound = false; + for (ListBlobItem blobItem : container.listBlobs()) { + CloudPageBlob blob3 = (CloudPageBlob) blobItem; + + if (blob.getName().equals(blobName) && !pageBlobWithTierFound) { + // Check that the blob is found exactly once + assertEquals(PremiumPageBlobTier.P40, blob3.properties.getPremiumPageBlobTier()); + assertFalse(blob3.getProperties().getInferredBlobTier()); + pageBlobWithTierFound = true; + } else if (blob.getName().equals(blobName)) { + fail("Page blob found twice"); + } + } + + assertTrue(pageBlobWithTierFound); + + try + { + CloudPageBlob blob4 = container.getPageBlobReference("blob4"); + blob4.create(256 * (long)Constants.GB); + blob4.uploadPremiumPageBlobTier(PremiumPageBlobTier.P6); + fail("Expected failure when setting blob tier size to be less than content length"); + } + catch (StorageException e) + { + assertEquals("Specified blob tier size limit cannot be less than content length.", e.getMessage()); + } + + try + { + blob2.uploadPremiumPageBlobTier(PremiumPageBlobTier.P4); + fail("Expected failure when attempted to set the tier to a lower value than previously"); + } + catch (StorageException e) + { + assertEquals("A higher blob tier has already been explicitly set.", e.getMessage()); + } + } + finally { + container.deleteIfExists(); + } + } + + @Test + @Category(PremiumBlobTests.class) + public void testCloudPageBlobSetBlobTierOnCopy() throws URISyntaxException, StorageException, InterruptedException { + CloudBlobContainer container = BlobTestHelper.getRandomPremiumBlobContainerReference(); + try { + container.create(); + CloudPageBlob source = container.getPageBlobReference("source"); + source.create(1024, PremiumPageBlobTier.P10, null, null, null); + + // copy to larger disk + CloudPageBlob copy = container.getPageBlobReference("copy"); + String copyId = copy.startCopy(TestHelper.defiddler(source.getUri()), PremiumPageBlobTier.P30, null, null, null, null); + assertEquals(BlobType.PAGE_BLOB, copy.getProperties().getBlobType()); + assertEquals(PremiumPageBlobTier.P30, copy.getProperties().getPremiumPageBlobTier()); + assertEquals(PremiumPageBlobTier.P10, source.getProperties().getPremiumPageBlobTier()); + assertFalse(source.getProperties().getInferredBlobTier()); + assertFalse(copy.getProperties().getInferredBlobTier()); + BlobTestHelper.waitForCopy(copy); + + CloudPageBlob copyRef = container.getPageBlobReference("copy"); + copyRef.downloadAttributes(); + assertEquals(PremiumPageBlobTier.P30, copyRef.getProperties().getPremiumPageBlobTier()); + assertNull(copyRef.getProperties().getInferredBlobTier()); + + // copy where source does not have a tier + CloudPageBlob source2 = container.getPageBlobReference("source2"); + source2.create(1024); + + CloudPageBlob copy3 = container.getPageBlobReference("copy3"); + String copyId3 = copy3.startCopy(TestHelper.defiddler(source2.getUri()), PremiumPageBlobTier.P60, null ,null ,null, null); + assertEquals(BlobType.PAGE_BLOB, copy3.getProperties().getBlobType()); + assertEquals(PremiumPageBlobTier.P60, copy3.getProperties().getPremiumPageBlobTier()); + assertNull(source2.getProperties().getPremiumPageBlobTier()); + assertNull(source2.getProperties().getInferredBlobTier()); + assertFalse(copy3.getProperties().getInferredBlobTier()); + } + finally { + container.deleteIfExists(); + } + } } diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java index 911d0eed86dbc..f819c63c57a3a 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java @@ -641,7 +641,7 @@ public static class HeaderConstants { /** * The current storage version header value. */ - public static final String TARGET_STORAGE_VERSION = "2016-05-31"; + public static final String TARGET_STORAGE_VERSION = "2017-04-17"; /** * The header that specifies the next visible time for a queue message. @@ -893,7 +893,12 @@ public static class QueryConstants { * XML element for an access policy. */ public static final String ACCESS_POLICY = "AccessPolicy"; - + + /** + * XML element for access tier. + */ + public static final String ACCESS_TIER = "AccessTier"; + /** * Buffer width used to copy data to output streams. */ diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobConstants.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobConstants.java index 6df87f96dde50..603e513644cb3 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobConstants.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobConstants.java @@ -20,12 +20,21 @@ * Holds the Constants used for the Blob Service. */ final class BlobConstants { - + + /** + * The header that specifies the access tier header. + */ + public static final String ACCESS_TIER_HEADER = Constants.PREFIX_FOR_STORAGE_HEADER + "access-tier"; + + /** + * The header that specifies if the access tier is inferred. + */ + public static final String ACCESS_TIER_INFERRED_HEADER = Constants.PREFIX_FOR_STORAGE_HEADER + "access-tier-inferred"; /** * Specifies the append blob type. */ public static final String APPEND_BLOB = "AppendBlob"; - + /** * XML element for authentication error details. */ diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobListHandler.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobListHandler.java index 7cb3f3ba67ddd..1898af48aad70 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobListHandler.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobListHandler.java @@ -324,5 +324,10 @@ else if (Constants.COPY_DESTINATION_SNAPSHOT_ID_ELEMENT.equals(currentNode)) { } this.copyState.setCopyDestinationSnapshotID(value); } + else if (Constants.ACCESS_TIER.equals(currentNode)) { + PremiumPageBlobTier premiumPageBlobTier = PremiumPageBlobTier.parse(value); + this.properties.setPremiumPageBlobTier(premiumPageBlobTier); + this.properties.setBlobTierInferredTier(false); + } } } \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobProperties.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobProperties.java index b5519acbbb7cb..0c227c48945bb 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobProperties.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobProperties.java @@ -117,6 +117,16 @@ public final class BlobProperties { */ private boolean isIncrementalCopy; + /** + * Represents the premium page blob tier. + */ + private PremiumPageBlobTier premiumPageBlobTier; + + /** + * Represents whether or not the blob tier is inferred. + */ + private Boolean isBlobTierInferredTier; + /** * Creates an instance of the BlobProperties class. */ @@ -150,6 +160,8 @@ public BlobProperties(final BlobProperties other) { this.pageBlobSequenceNumber = other.pageBlobSequenceNumber; this.serverEncrypted = other.serverEncrypted; this.isIncrementalCopy = other.isIncrementalCopy; + this.premiumPageBlobTier = other.premiumPageBlobTier; + this.isBlobTierInferredTier = other.isBlobTierInferredTier; } /** @@ -271,6 +283,13 @@ public Date getLastModified() { return this.lastModified; } + /** + * Gets a value indicating if the tier of the premium page blob has been inferred. + * + * @return A {@Link java.lang.Boolean} object which represents if the blob tier was inferred. + */ + public Boolean getInferredBlobTier() { return this.isBlobTierInferredTier; } + /** * Gets the lease status for the blob. * @@ -315,7 +334,16 @@ public long getLength() { public Long getPageBlobSequenceNumber() { return this.pageBlobSequenceNumber; } - + + /** + * If using a premium account and the blob is a page blob, gets the tier of the blob. + * @return A {@link PremiumPageBlobTier} object which represents the tier of the blob + * or null if the tier has not been set. + */ + public PremiumPageBlobTier getPremiumPageBlobTier() { + return this.premiumPageBlobTier; + } + /** * Gets the blob's server-side encryption status; * @@ -512,4 +540,22 @@ protected void setServerEncrypted(boolean serverEncrypted) { protected void setIncrementalCopy(boolean isIncrementalCopy) { this.isIncrementalCopy = isIncrementalCopy; } + + /** + * Sets the tier of the page blob. This is only supported for premium accounts. + * @param premiumPageBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + */ + protected void setPremiumPageBlobTier(PremiumPageBlobTier premiumPageBlobTier) { + this.premiumPageBlobTier = premiumPageBlobTier; + } + + /** + * Sets whether the blob tier is inferred. + * @param isBlobTierInferredTier + * A {@Link java.lang.Boolean} which specifies if the blob tier is inferred. + */ + protected void setBlobTierInferredTier(Boolean isBlobTierInferredTier) { + this.isBlobTierInferredTier = isBlobTierInferredTier; + } } \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java index 7fe4e69b9c22d..8be31ddcf6aa4 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java @@ -54,6 +54,8 @@ final class BlobRequest { private static final String SNAPSHOTS_QUERY_ELEMENT_NAME = "snapshots"; + private static final String TIER_QUERY_ELEMENT_NAME = "tier"; + private static final String UNCOMMITTED_BLOBS_QUERY_ELEMENT_NAME = "uncommittedblobs"; /** @@ -219,6 +221,8 @@ public static HttpURLConnection appendBlock(final URI uri, final BlobRequestOpti * The snapshot version, if the source blob is a snapshot. * @param incrementalCopy * A boolean indicating whether or not this is an incremental copy. + * @param premiumPageBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. * @return a HttpURLConnection configured for the operation. * @throws StorageException * an exception representing any error which occurred during the operation. @@ -229,7 +233,7 @@ public static HttpURLConnection appendBlock(final URI uri, final BlobRequestOpti public static HttpURLConnection copyFrom(final URI uri, final BlobRequestOptions blobOptions, final OperationContext opContext, final AccessCondition sourceAccessCondition, final AccessCondition destinationAccessCondition, String source, final String sourceSnapshotID, - final boolean incrementalCopy) + final boolean incrementalCopy, final PremiumPageBlobTier premiumPageBlobTier) throws StorageException, IOException, URISyntaxException { if (sourceSnapshotID != null) { @@ -252,6 +256,10 @@ public static HttpURLConnection copyFrom(final URI uri, final BlobRequestOptions request.setRequestProperty(Constants.HeaderConstants.COPY_SOURCE_HEADER, source); + if (premiumPageBlobTier != null) { + request.setRequestProperty(BlobConstants.ACCESS_TIER_HEADER, String.valueOf(premiumPageBlobTier)); + } + if (sourceAccessCondition != null) { sourceAccessCondition.applySourceConditionToRequest(request); } @@ -1146,6 +1154,44 @@ public static HttpURLConnection listContainers(final URI uri, final BlobRequestO public static HttpURLConnection putBlob(final URI uri, final BlobRequestOptions blobOptions, final OperationContext opContext, final AccessCondition accessCondition, final BlobProperties properties, final BlobType blobType, final long pageBlobSize) throws IOException, URISyntaxException, StorageException { + return BlobRequest.putBlob(uri, blobOptions, opContext, accessCondition, properties, blobType, pageBlobSize, null /* premiumPageBlobTier */); + } + + /** + * Constructs a HttpURLConnection to upload a blob. Sign with blob length, or -1 for pageblob create. + * + * @param uri + * A java.net.URI object that specifies the absolute URI. + * @param blobOptions + * A {@link BlobRequestOptions} object that specifies execution options such as retry policy and timeout + * settings for the operation. Specify null to use the request options specified on the + * {@link CloudBlobClient}. + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * @param accessCondition + * An {@link AccessCondition} object that represents the access conditions for the blob. + * @param properties + * The properties to set for the blob. + * @param blobType + * The type of the blob. + * @param pageBlobSize + * For a page blob, the size of the blob. This parameter is ignored for block blobs. + * @param premiumPageBlobTier + * A {@link PremiumPageBlobTier} object representing the tier to set. + * @return a HttpURLConnection to use to perform the operation. + * @throws IOException + * if there is an error opening the connection + * @throws URISyntaxException + * if the resource URI is invalid + * @throws StorageException + * an exception representing any error which occurred during the operation. + * @throws IllegalArgumentException + */ + public static HttpURLConnection putBlob(final URI uri, final BlobRequestOptions blobOptions, + final OperationContext opContext, final AccessCondition accessCondition, final BlobProperties properties, + final BlobType blobType, final long pageBlobSize, final PremiumPageBlobTier premiumPageBlobTier) throws IOException, URISyntaxException, StorageException { if (blobType == BlobType.UNSPECIFIED) { throw new IllegalArgumentException(SR.BLOB_TYPE_NOT_DEFINED); } @@ -1165,6 +1211,11 @@ public static HttpURLConnection putBlob(final URI uri, final BlobRequestOptions request.setRequestProperty(BlobConstants.BLOB_TYPE_HEADER, BlobConstants.PAGE_BLOB); request.setRequestProperty(BlobConstants.SIZE, String.valueOf(pageBlobSize)); + if (premiumPageBlobTier != null) + { + request.setRequestProperty(BlobConstants.ACCESS_TIER_HEADER, String.valueOf(premiumPageBlobTier)); + } + properties.setLength(pageBlobSize); } else if (blobType == BlobType.BLOCK_BLOB){ @@ -1227,6 +1278,48 @@ public static HttpURLConnection putBlock(final URI uri, final BlobRequestOptions return request; } + + /** + * Constructs a HttpURLConnection to set the the tier on a page blob. + * This API is only supported for premium accounts. + * + * @param uri + * A java.net.URI object that specifies the absolute URI. + * @param blobOptions + * A {@link BlobRequestOptions} object that specifies execution options such as retry policy and timeout + * settings for the operation. Specify null to use the request options specified on the + * {@link CloudBlobClient}. + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object representing the tier to set. + * @return a HttpURLConnection to use to perform the operation. + * @throws IOException + * if there is an error opening the connection + * @throws URISyntaxException + * if the resource URI is invalid + * @throws StorageException + * an exception representing any error which occurred during the operation. + * @throws IllegalArgumentException + */ + public static HttpURLConnection setBlobTier(final URI uri, final BlobRequestOptions blobOptions, + final OperationContext opContext, final String premiumBlobTier) + throws IOException, URISyntaxException, StorageException { + final UriQueryBuilder builder = new UriQueryBuilder(); + builder.add(Constants.QueryConstants.COMPONENT, TIER_QUERY_ELEMENT_NAME); + + final HttpURLConnection request = createURLConnection(uri, builder, blobOptions, opContext); + + request.setDoOutput(true); + request.setRequestMethod(Constants.HTTP_PUT); + request.setFixedLengthStreamingMode(0); + request.setRequestProperty(Constants.HeaderConstants.CONTENT_LENGTH, "0"); + request.setRequestProperty(BlobConstants.ACCESS_TIER_HEADER, premiumBlobTier); + + return request; + } /** * Constructs a HttpURLConnection to write a blob by specifying the list of block IDs that make up the blob. Sign diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobResponse.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobResponse.java index c31f05ab898a9..0e26efa41847d 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobResponse.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobResponse.java @@ -124,6 +124,26 @@ else if (!Utility.isNullOrEmpty(xContentLengthHeader)) { properties.setAppendBlobCommittedBlockCount(Integer.parseInt(comittedBlockCount)); } + // Get the tier of the blob + final String premiumBlobTierString = request.getHeaderField(BlobConstants.ACCESS_TIER_HEADER); + + if (properties.getBlobType().equals(BlobType.PAGE_BLOB)) + { + PremiumPageBlobTier premiumPageBlobTier = PremiumPageBlobTier.parse(premiumBlobTierString); + properties.setPremiumPageBlobTier(premiumPageBlobTier); + } + else if (properties.getBlobType().equals(BlobType.UNSPECIFIED)) { + PremiumPageBlobTier premiumPageBlobTier = PremiumPageBlobTier.parse(premiumBlobTierString); + if (!premiumPageBlobTier.equals(PremiumPageBlobTier.UNKNOWN)) { + properties.setPremiumPageBlobTier(premiumPageBlobTier); + } + } + + final String tierInferredString = request.getHeaderField(BlobConstants.ACCESS_TIER_INFERRED_HEADER); + if (!Utility.isNullOrEmpty(tierInferredString)) { + properties.setBlobTierInferredTier(Boolean.parseBoolean(tierInferredString)); + } + final String incrementalCopyHeaderString = request.getHeaderField(Constants.HeaderConstants.INCREMENTAL_COPY); if (!Utility.isNullOrEmpty(incrementalCopyHeaderString)) { diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java index 961d211f256f9..f4b700d980545 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java @@ -681,6 +681,43 @@ public final String startCopy(final URI source) throws StorageException { */ @DoesServiceRequest public final String startCopy(final URI source, final AccessCondition sourceAccessCondition, + final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext) + throws StorageException { + return this.startCopy(source, null /* premiumPageBlobTier */, sourceAccessCondition, destinationAccessCondition, options, opContext); + } + + /** + * Requests the service to start copying a URI's contents, properties, and metadata to a new blob, using the + * specified premium page blob tier, access conditions, lease ID, request options, and operation context. + *

+ * Note: Setting the premiumPageBlobTier is only supported for premium accounts. + *

+ * @param source + * A java.net.URI The source URI. URIs for resources outside of Azure + * may only be copied into block blobs. + * @param premiumPageBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param sourceAccessCondition + * An {@link AccessCondition} object that represents the access conditions for the source. + * @param destinationAccessCondition + * An {@link AccessCondition} object that represents the access conditions for the destination. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. + * Specifying null will use the default request options from the associated + * service client ({@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. + * This object is used to track requests to the storage service, and to provide additional + * runtime information about the operation. + * + * @return A String which represents the copy ID associated with the copy operation. + * + * @throws StorageException + * If a storage service error occurred. + * + */ + @DoesServiceRequest + protected final String startCopy(final URI source, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition, final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException { if (opContext == null) { @@ -691,12 +728,12 @@ public final String startCopy(final URI source, final AccessCondition sourceAcce options = BlobRequestOptions.populateAndApplyDefaults(options, this.properties.getBlobType(), this.blobServiceClient); return ExecutionEngine.executeWithRetry(this.blobServiceClient, this, - this.startCopyImpl(source, false /* incrementalCopy */, sourceAccessCondition, destinationAccessCondition, options), + this.startCopyImpl(source, false /* incrementalCopy */, premiumPageBlobTier, sourceAccessCondition, destinationAccessCondition, options), options.getRetryPolicyFactory(), opContext); } protected StorageRequest startCopyImpl( - final URI source, final boolean incrementalCopy, final AccessCondition sourceAccessCondition, + final URI source, final boolean incrementalCopy, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition, final AccessCondition destinationAccessCondition, final BlobRequestOptions options) { final StorageRequest putRequest = @@ -708,7 +745,7 @@ public HttpURLConnection buildRequest(CloudBlobClient client, CloudBlob blob, Op // toASCIIString() must be used in order to appropriately encode the URI return BlobRequest.copyFrom(blob.getTransformedAddress(context).getUri(this.getCurrentLocation()), options, context, sourceAccessCondition, destinationAccessCondition, source.toASCIIString(), - blob.snapshotID, incrementalCopy); + blob.snapshotID, incrementalCopy, premiumPageBlobTier); } @Override @@ -732,6 +769,10 @@ public String preProcessResponse(CloudBlob blob, CloudBlobClient client, Operati blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); blob.properties.setCopyState(BlobResponse.getCopyState(this.getConnection())); + blob.properties.setPremiumPageBlobTier(premiumPageBlobTier); + if (premiumPageBlobTier != null) { + blob.properties.setBlobTierInferredTier(false); + } return blob.properties.getCopyState().getCopyId(); } @@ -2888,7 +2929,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation * A {@link StorageUri} object which represents the resource URI. * @param delimiter * A String which specifies the directory delimiter to use. - * @param usePathStyleUris + * @param container * A {@link CloudBlobContainer} object which represents the blob container. * * @return A String which represents the parent address for a blob URI. diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java index 53863bed5171a..dab9c59827cfd 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java @@ -14,9 +14,7 @@ */ package com.microsoft.azure.storage.blob; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -214,6 +212,41 @@ public final String startCopy(final CloudPageBlob sourceBlob) throws StorageExce */ @DoesServiceRequest public final String startCopy(final CloudPageBlob sourceBlob, final AccessCondition sourceAccessCondition, + final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext) + throws StorageException, URISyntaxException { + return this.startCopy(sourceBlob, null /* premiumBlobTier */, sourceAccessCondition, destinationAccessCondition, options, opContext); + } + + /** + * Requests the service to start copying a blob's contents, properties, and metadata to a new blob, using the + * specified blob tier, access conditions, lease ID, request options, and operation context. + * + * @param sourceBlob + * A CloudPageBlob object that represents the source blob to copy. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param sourceAccessCondition + * An {@link AccessCondition} object that represents the access conditions for the source blob. + * @param destinationAccessCondition + * An {@link AccessCondition} object that represents the access conditions for the destination blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @return A String which represents the copy ID associated with the copy operation. + * + * @throws StorageException + * If a storage service error occurred. + * @throws URISyntaxException + * + */ + @DoesServiceRequest + public final String startCopy(final CloudPageBlob sourceBlob, final PremiumPageBlobTier premiumBlobTier, final AccessCondition sourceAccessCondition, final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { Utility.assertNotNull("sourceBlob", sourceBlob); @@ -224,7 +257,7 @@ public final String startCopy(final CloudPageBlob sourceBlob, final AccessCondit source = sourceBlob.getServiceClient().getCredentials().transformUri(sourceBlob.getSnapshotQualifiedUri()); } - return this.startCopy(source, sourceAccessCondition, destinationAccessCondition, options, opContext); + return this.startCopy(source, premiumBlobTier, sourceAccessCondition, destinationAccessCondition, options, opContext); } /** @@ -343,7 +376,7 @@ public final String startIncrementalCopy(final URI sourceSnapshot, options = BlobRequestOptions.populateAndApplyDefaults(options, this.properties.getBlobType(), this.blobServiceClient); return ExecutionEngine.executeWithRetry(this.blobServiceClient, this, - this.startCopyImpl(sourceSnapshot, true /* incrementalCopy */, null /* sourceAccesCondition */, + this.startCopyImpl(sourceSnapshot, true /* incrementalCopy */, null /* premiumPageBlobTier */, null /* sourceAccesCondition */, destinationAccessCondition, options), options.getRetryPolicyFactory(), opContext); } @@ -435,6 +468,36 @@ public void create(final long length) throws StorageException { this.create(length, null /* accessCondition */, null /* options */, null /* opContext */); } + /** + * Creates a page blob using the specified request options and operation context. If the blob already exists, + * this will replace it. To instead throw an error if the blob already exists, use + * {@link AccessCondition#generateIfNotExistsCondition()}. + * + * @param length + * A long which represents the size, in bytes, of the page blob. + * @param accessCondition + * An {@link AccessCondition} object which represents the access conditions for the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object which represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @throws IllegalArgumentException + * If the length is not a multiple of 512. + * + * @throws StorageException + * If a storage service error occurred. + */ + @DoesServiceRequest + public void create(final long length, final AccessCondition accessCondition, BlobRequestOptions options, + OperationContext opContext) throws StorageException { + this.create(length, null /* premiumBlobTier */, accessCondition, options, opContext); + } + /** * Creates a page blob using the specified request options and operation context. If the blob already exists, * this will replace it. To instead throw an error if the blob already exists, use @@ -442,6 +505,8 @@ public void create(final long length) throws StorageException { * * @param length * A long which represents the size, in bytes, of the page blob. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. * @param accessCondition * An {@link AccessCondition} object which represents the access conditions for the blob. * @param options @@ -460,7 +525,7 @@ public void create(final long length) throws StorageException { * If a storage service error occurred. */ @DoesServiceRequest - public void create(final long length, final AccessCondition accessCondition, BlobRequestOptions options, + public void create(final long length, final PremiumPageBlobTier premiumBlobTier, final AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException { assertNoWriteOperationForSnapshot(); @@ -475,10 +540,10 @@ public void create(final long length, final AccessCondition accessCondition, Blo options = BlobRequestOptions.populateAndApplyDefaults(options, BlobType.PAGE_BLOB, this.blobServiceClient); ExecutionEngine.executeWithRetry(this.blobServiceClient, this, - this.createImpl(length, accessCondition, options), options.getRetryPolicyFactory(), opContext); + this.createImpl(length, premiumBlobTier, accessCondition, options), options.getRetryPolicyFactory(), opContext); } - private StorageRequest createImpl(final long length, + private StorageRequest createImpl(final long length, final PremiumPageBlobTier premiumBlobTier, final AccessCondition accessCondition, final BlobRequestOptions options) { final StorageRequest putRequest = new StorageRequest( options, this.getStorageUri()) { @@ -487,7 +552,7 @@ private StorageRequest createImpl(final long l public HttpURLConnection buildRequest(CloudBlobClient client, CloudBlob blob, OperationContext context) throws Exception { return BlobRequest.putBlob(blob.getTransformedAddress(context).getUri(this.getCurrentLocation()), - options, context, accessCondition, blob.properties, BlobType.PAGE_BLOB, length); + options, context, accessCondition, blob.properties, BlobType.PAGE_BLOB, length, premiumBlobTier); } @Override @@ -512,6 +577,11 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); blob.getProperties().setLength(length); + blob.getProperties().setPremiumPageBlobTier(premiumBlobTier); + if (premiumBlobTier != null) { + blob.getProperties().setBlobTierInferredTier(false); + } + return null; } @@ -806,7 +876,7 @@ public List postProcessResponse(HttpURLConnection connection, Clo @DoesServiceRequest public BlobOutputStream openWriteExisting() throws StorageException { return this - .openOutputStreamInternal(null /* length */, null /* accessCondition */, null /* options */, null /* opContext */); + .openOutputStreamInternal(null /* length */, null /* premiumBlobTier */,null /* accessCondition */, null /* options */, null /* opContext */); } /** @@ -832,7 +902,7 @@ public BlobOutputStream openWriteExisting() throws StorageException { @DoesServiceRequest public BlobOutputStream openWriteExisting(AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException { - return this.openOutputStreamInternal(null /* length */, accessCondition, options, opContext); + return this.openOutputStreamInternal(null /* length */, null /* premiumBlobTier */, accessCondition, options, opContext); } /** @@ -855,7 +925,40 @@ public BlobOutputStream openWriteExisting(AccessCondition accessCondition, BlobR @DoesServiceRequest public BlobOutputStream openWriteNew(final long length) throws StorageException { return this - .openOutputStreamInternal(length, null /* accessCondition */, null /* options */, null /* opContext */); + .openOutputStreamInternal(length, null /* premiumBlobTier */, null /* accessCondition */, null /* options */, null /* opContext */); + } + + /** + * Opens an output stream object to write data to the page blob, using the specified lease ID, request options and + * operation context. The page blob does not need to yet exist and will be created with the length specified.If the + * blob already exists on the service, it will be overwritten. + *

+ * To avoid overwriting and instead throw an error, please pass in an {@link AccessCondition} generated using + * {@link AccessCondition#generateIfNotExistsCondition()}. + * + * @param length + * A long which represents the length, in bytes, of the stream to create. This value must be + * a multiple of 512. + * @param accessCondition + * An {@link AccessCondition} object which represents the access conditions for the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object which represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @return A {@link BlobOutputStream} object used to write data to the blob. + * + * @throws StorageException + * If a storage service error occurred. + */ + @DoesServiceRequest + public BlobOutputStream openWriteNew(final long length, AccessCondition accessCondition, + BlobRequestOptions options, OperationContext opContext) throws StorageException { + return openOutputStreamInternal(length, null /* premiumBlobTier */, accessCondition, options, opContext); } /** @@ -869,6 +972,8 @@ public BlobOutputStream openWriteNew(final long length) throws StorageException * @param length * A long which represents the length, in bytes, of the stream to create. This value must be * a multiple of 512. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. * @param accessCondition * An {@link AccessCondition} object which represents the access conditions for the blob. * @param options @@ -886,9 +991,9 @@ public BlobOutputStream openWriteNew(final long length) throws StorageException * If a storage service error occurred. */ @DoesServiceRequest - public BlobOutputStream openWriteNew(final long length, AccessCondition accessCondition, + public BlobOutputStream openWriteNew(final long length, final PremiumPageBlobTier premiumBlobTier, AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException { - return openOutputStreamInternal(length, accessCondition, options, opContext); + return openOutputStreamInternal(length, premiumBlobTier, accessCondition, options, opContext); } /** @@ -900,6 +1005,8 @@ public BlobOutputStream openWriteNew(final long length, AccessCondition accessCo * A long which represents the length, in bytes, of the stream to create. This value must be * a multiple of 512 or null if the * page blob already exists. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. * @param accessCondition * An {@link AccessCondition} object which represents the access conditions for the blob. * @param options @@ -916,7 +1023,7 @@ public BlobOutputStream openWriteNew(final long length, AccessCondition accessCo * @throws StorageException * If a storage service error occurred. */ - private BlobOutputStream openOutputStreamInternal(Long length, AccessCondition accessCondition, + private BlobOutputStream openOutputStreamInternal(Long length, PremiumPageBlobTier premiumBlobTier, AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException { if (opContext == null) { opContext = new OperationContext(); @@ -943,9 +1050,8 @@ private BlobOutputStream openOutputStreamInternal(Long length, AccessCondition a throw new IllegalArgumentException(SR.INVALID_PAGE_BLOB_LENGTH); } - this.create(length, accessCondition, options, opContext); + this.create(length, premiumBlobTier, accessCondition, options, opContext); } - else { if (options.getEncryptionPolicy() != null) { throw new IllegalArgumentException(SR.ENCRYPTION_NOT_SUPPORTED_FOR_EXISTING_BLOBS); @@ -1152,6 +1258,71 @@ public Void preProcessResponse(CloudPageBlob blob, CloudBlobClient client, Opera return putRequest; } + /** + * Uploads a blob from data in a byte array. If the blob already exists on the service, it will be overwritten. + * + * @param buffer + * A byte array which represents the data to write to the blob. + * @param offset + * A int which represents the offset of the byte array from which to start the data upload. + * @param length + * An int which represents the number of bytes to upload from the input buffer. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param accessCondition + * An {@link AccessCondition} object that represents the access conditions for the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @throws StorageException + * If a storage service error occurred. + * @throws IOException + */ + public void uploadFromByteArray(final byte[] buffer, final int offset, final int length, final PremiumPageBlobTier premiumBlobTier, + final AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) + throws StorageException, IOException { + ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer, offset, length); + this.upload(inputStream, length, premiumBlobTier, accessCondition, options, opContext); + inputStream.close(); + } + + /** + * Uploads a blob from a file. If the blob already exists on the service, it will be overwritten. + * + * @param path + * A String which represents the path to the file to be uploaded. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param accessCondition + * An {@link AccessCondition} object that represents the access conditions for the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @throws StorageException + * If a storage service error occurred. + * @throws IOException + */ + public void uploadFromFile(final String path, final PremiumPageBlobTier premiumBlobTier, final AccessCondition accessCondition, BlobRequestOptions options, + OperationContext opContext) throws StorageException, IOException { + File file = new File(path); + long fileLength = file.length(); + InputStream inputStream = new BufferedInputStream(new FileInputStream(file)); + this.upload(inputStream, fileLength, premiumBlobTier, accessCondition, options, opContext); + inputStream.close(); + } + /** * Uploads the source stream data to the page blob. If the blob already exists on the service, it will be * overwritten. @@ -1170,13 +1341,13 @@ public Void preProcessResponse(CloudPageBlob blob, CloudBlobClient client, Opera @Override @DoesServiceRequest public void upload(final InputStream sourceStream, final long length) throws StorageException, IOException { - this.upload(sourceStream, length, null /* accessCondition */, null /* options */, null /* opContext */); + this.upload(sourceStream, length, null /* premiumBlobTier */, null /* accessCondition */, null /* options */, null /* opContext */); } /** * Uploads the source stream data to the page blob using the specified lease ID, request options, and operation * context. If the blob already exists on the service, it will be overwritten. - * + * * @param sourceStream * An {@link InputStream} object to read from. * @param length @@ -1192,7 +1363,7 @@ public void upload(final InputStream sourceStream, final long length) throws Sto * An {@link OperationContext} object which represents the context for the current operation. This object * is used to track requests to the storage service, and to provide additional runtime information about * the operation. - * + * * @throws IOException * If an I/O exception occurred. * @throws StorageException @@ -1201,6 +1372,39 @@ public void upload(final InputStream sourceStream, final long length) throws Sto @Override @DoesServiceRequest public void upload(final InputStream sourceStream, final long length, final AccessCondition accessCondition, + BlobRequestOptions options, OperationContext opContext) throws StorageException, IOException { + this.upload(sourceStream, length, null /* premiumBlobTier*/, accessCondition, options, opContext); + } + + /** + * Uploads the source stream data to the page blob using the specified lease ID, request options, and operation + * context. If the blob already exists on the service, it will be overwritten. + * + * @param sourceStream + * An {@link InputStream} object to read from. + * @param length + * A long which represents the length, in bytes, of the stream data. This must be great than + * zero and a multiple of 512. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param accessCondition + * An {@link AccessCondition} object which represents the access conditions for the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object which represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * + * @throws IOException + * If an I/O exception occurred. + * @throws StorageException + * If a storage service error occurred. + */ + @DoesServiceRequest + public void upload(final InputStream sourceStream, final long length, final PremiumPageBlobTier premiumBlobTier, final AccessCondition accessCondition, BlobRequestOptions options, OperationContext opContext) throws StorageException, IOException { assertNoWriteOperationForSnapshot(); @@ -1224,7 +1428,7 @@ public void upload(final InputStream sourceStream, final long length, final Acce sourceStream.mark(Constants.MAX_MARK_LENGTH); } - final BlobOutputStream streamRef = this.openWriteNew(length, accessCondition, options, opContext); + final BlobOutputStream streamRef = this.openWriteNew(length, premiumBlobTier, accessCondition, options, opContext); try { streamRef.write(sourceStream, length); } @@ -1365,4 +1569,85 @@ public void setStreamWriteSizeInBytes(final int streamWriteSizeInBytes) { this.streamWriteSizeInBytes = streamWriteSizeInBytes; } + + /** + * Sets the blob tier on a page blob on a premium storage account. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @throws StorageException + * If a storage service error occurred. + */ + @DoesServiceRequest + public void uploadPremiumPageBlobTier(final PremiumPageBlobTier premiumBlobTier) throws StorageException { + this.uploadPremiumPageBlobTier(premiumBlobTier, null /* options */, null /* opContext */); + } + + /** + * Sets the tier on a page blob on a premium storage account. + * @param premiumBlobTier + * A {@link PremiumPageBlobTier} object which represents the tier of the blob. + * @param options + * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying + * null will use the default request options from the associated service client ( + * {@link CloudBlobClient}). + * @param opContext + * An {@link OperationContext} object which represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * @throws StorageException + * If a storage service error occurred. + */ + @DoesServiceRequest + public void uploadPremiumPageBlobTier(final PremiumPageBlobTier premiumBlobTier, BlobRequestOptions options, + OperationContext opContext) throws StorageException { + assertNoWriteOperationForSnapshot(); + Utility.assertNotNull("premiumBlobTier", premiumBlobTier); + + if (opContext == null) { + opContext = new OperationContext(); + } + + options = BlobRequestOptions.populateAndApplyDefaults(options, BlobType.PAGE_BLOB, this.blobServiceClient); + + ExecutionEngine.executeWithRetry(this.blobServiceClient, this, + this.uploadPremiumPageBlobTierImpl(premiumBlobTier, options), options.getRetryPolicyFactory(), opContext); + } + + private StorageRequest uploadPremiumPageBlobTierImpl(final PremiumPageBlobTier premiumBlobTier, + final BlobRequestOptions options) { + final StorageRequest setTierRequest = new StorageRequest( + options, this.getStorageUri()) { + + @Override + public HttpURLConnection buildRequest(CloudBlobClient client, CloudBlob blob, OperationContext context) + throws Exception { + return BlobRequest.setBlobTier(blob.getTransformedAddress(context).getUri(this.getCurrentLocation()), options, context, premiumBlobTier.toString()); + } + + @Override + public void signRequest(HttpURLConnection connection, CloudBlobClient client, OperationContext context) + throws Exception { + StorageRequest.signBlobQueueAndFileRequest(connection, client, 0L, context); + } + + @Override + public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, OperationContext context) + throws Exception { + if (this.getResult().getStatusCode() != HttpURLConnection.HTTP_OK) { + this.setNonExceptionedRetryableFailure(true); + return null; + } + + blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); + this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); + blob.properties.setPremiumPageBlobTier(premiumBlobTier); + blob.properties.setBlobTierInferredTier(false); + + return null; + } + + }; + + return setTierRequest; + } } \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/PremiumPageBlobTier.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/PremiumPageBlobTier.java new file mode 100644 index 0000000000000..590a66c334f77 --- /dev/null +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/PremiumPageBlobTier.java @@ -0,0 +1,112 @@ +/** + * Copyright Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.microsoft.azure.storage.blob; + +import java.util.Locale; + +import com.microsoft.azure.storage.core.Utility; + +/** + * The tier of the page blob. + * Please take a look at https://docs.microsoft.com/en-us/azure/storage/storage-premium-storage#scalability-and-performance-targets + * for detailed information on the corresponding IOPS and throughput per PremiumPageBlobTier. + */ +public enum PremiumPageBlobTier { + /** + * The tier is not recognized by this version of the library. + */ + UNKNOWN, + + /** + * P4 Tier + */ + P4, + + /** + * P6 Tier + */ + P6, + + /** + * P10 Tier + */ + P10, + + /** + * P20 Tier + */ + P20, + + /** + * P30 Tier + */ + P30, + + /** + * P40 Tier + */ + P40, + + /** + * P50 Tier + */ + P50, + + /** + * P60 Tier + */ + P60; + + /** + * Parses a premium page blob tier from the given string. + * + * @param premiumBlobTierString + * A String which contains the premium page blob tier to parse. + * + * @return A PremiumPageBlobTier value that represents the premium page blob tier. + */ + protected static PremiumPageBlobTier parse(final String premiumBlobTierString) { + if (Utility.isNullOrEmpty(premiumBlobTierString)) { + return UNKNOWN; + } + else if ("p4".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P4; + } + else if ("p6".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P6; + } + else if ("p10".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P10; + } + else if ("p20".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P20; + } + else if ("p30".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P30; + } + else if ("p40".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P40; + } + else if ("p50".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P50; + } + else if ("p60".equals(premiumBlobTierString.toLowerCase(Locale.US))) { + return P60; + } + else { + return UNKNOWN; + } + } +} \ No newline at end of file From bb9d7df951695217297b69bd53c17a4bbf5826e7 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Tue, 11 Jul 2017 10:58:19 -0700 Subject: [PATCH 5/7] Removed request encryption check from Set Tier API --- .../src/com/microsoft/azure/storage/blob/CloudPageBlob.java | 1 - 1 file changed, 1 deletion(-) diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java index 41335236e2d0f..8553b05198379 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudPageBlob.java @@ -1640,7 +1640,6 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation } blob.updateEtagAndLastModifiedFromResponse(this.getConnection()); - this.getResult().setRequestServiceEncrypted(CloudBlob.isServerRequestEncrypted(this.getConnection())); blob.properties.setPremiumPageBlobTier(premiumBlobTier); blob.properties.setBlobTierInferredTier(false); From 87c9f8a1dc59a81f8e99d23914ed74b50e19826c Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Wed, 12 Jul 2017 09:30:49 -0700 Subject: [PATCH 6/7] Updating version for 5.4.0 Release --- ChangeLog.txt | 2 +- README.md | 2 +- microsoft-azure-storage-samples/pom.xml | 2 +- .../src/com/microsoft/azure/storage/Constants.java | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index b75ba99726ddd..7fe17ea69bbf5 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,4 +1,4 @@ -2017.XX.XX Version X.X.X +2017.07.12 Version 5.4.0 * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire. * For Premium Accounts only, added support for getting and setting the tier on a page blob. The tier can also be set when creating or copying from an existing page blob. * Added support for server side encryption for File Service. diff --git a/README.md b/README.md index 4f6743cfde7bb..db606759878c6 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ To get the binaries of this library as distributed by Microsoft, ready for use w com.microsoft.azure azure-storage - 5.3.1 + 5.4.0 ``` diff --git a/microsoft-azure-storage-samples/pom.xml b/microsoft-azure-storage-samples/pom.xml index 8f4e5c09045f7..ad5b5f2e807c6 100644 --- a/microsoft-azure-storage-samples/pom.xml +++ b/microsoft-azure-storage-samples/pom.xml @@ -26,7 +26,7 @@ com.microsoft.azure azure-storage - 5.3.1 + 5.4.0 com.microsoft.azure diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java index f819c63c57a3a..278150e9e9f65 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java @@ -661,7 +661,7 @@ public static class HeaderConstants { /** * Specifies the value to use for UserAgent header. */ - public static final String USER_AGENT_VERSION = "5.3.1"; + public static final String USER_AGENT_VERSION = "5.4.0"; /** * The default type for content-type and accept diff --git a/pom.xml b/pom.xml index ac91bbaf98fe8..e36ef55894d49 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 com.microsoft.azure azure-storage - 5.3.1 + 5.4.0 jar Microsoft Azure Storage Client SDK From 05b4aa84c5afe706f278784f56112b93da825d31 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Thu, 13 Jul 2017 11:20:45 -0700 Subject: [PATCH 7/7] Updating changelog date and service version --- ChangeLog.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index 7fe17ea69bbf5..04d1a0e8cc285 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,4 +1,5 @@ -2017.07.12 Version 5.4.0 +2017.07.13 Version 5.4.0 + * Support for 2017-04-17 REST version. Please see our REST API documentation and blogs for information about the related added features. * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire. * For Premium Accounts only, added support for getting and setting the tier on a page blob. The tier can also be set when creating or copying from an existing page blob. * Added support for server side encryption for File Service.