From 21e68cce2c133463fcc4d74e06e3114c5e25ef65 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 29 Feb 2016 15:10:12 -0800 Subject: [PATCH 1/3] Add a versions option to BlobListOption --- .../main/java/com/google/gcloud/storage/Storage.java | 10 ++++++++++ .../com/google/gcloud/storage/StorageImplTest.java | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 4acc1dbcad07..4fa4ba3658be 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -700,6 +700,16 @@ public static BlobListOption recursive(boolean recursive) { return new BlobListOption(StorageRpc.Option.DELIMITER, recursive); } + /** + * If set to {@code true}, lists all versions of a blob. + * The default is {@code false}. + * + * @see Object Versioning + */ + public static BlobListOption versions(boolean versions) { + return new BlobListOption(StorageRpc.Option.VERSIONS, versions); + } + /** * Returns an option to specify the blob's fields to be returned by the RPC call. If this option * is not provided all blob's fields are returned. {@code BlobListOption.fields}) can be used to diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index b14dcd057438..612664de14ae 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -200,11 +200,14 @@ public class StorageImplTest { Storage.BlobListOption.prefix("prefix"); private static final Storage.BlobListOption BLOB_LIST_FIELDS = Storage.BlobListOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.MD5HASH); + private static final Storage.BlobListOption BLOB_LIST_VERSIONS = + Storage.BlobListOption.versions(false); private static final Storage.BlobListOption BLOB_LIST_EMPTY_FIELDS = Storage.BlobListOption.fields(); private static final Map BLOB_LIST_OPTIONS = ImmutableMap.of( StorageRpc.Option.MAX_RESULTS, BLOB_LIST_MAX_RESULT.value(), - StorageRpc.Option.PREFIX, BLOB_LIST_PREFIX.value()); + StorageRpc.Option.PREFIX, BLOB_LIST_PREFIX.value(), + StorageRpc.Option.VERSIONS, BLOB_LIST_VERSIONS.value()); private static final String PRIVATE_KEY_STRING = "MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoG" + "BAL2xolH1zrISQ8+GzOV29BNjjzq4/HIP8Psd1+cZb81vDklSF+95wB250MSE0BDc81pvIMwj5OmIfLg1NY6uB" @@ -650,7 +653,8 @@ public void testListBlobsWithOptions() { EasyMock.replay(storageRpcMock); initializeService(); ImmutableList blobList = ImmutableList.of(expectedBlob1, expectedBlob2); - Page page = storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX); + Page page = + storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX, BLOB_LIST_VERSIONS); assertEquals(cursor, page.nextPageCursor()); assertArrayEquals(blobList.toArray(), Iterables.toArray(page.values(), Blob.class)); } From 438896b96ded6829c3fe869a145ab3cf2c9bcaf9 Mon Sep 17 00:00:00 2001 From: Arie Ozarov Date: Mon, 29 Feb 2016 20:12:00 -0800 Subject: [PATCH 2/3] add list versioned blobs to integration tests --- .../storage/testing/RemoteGcsHelper.java | 5 +- .../gcloud/storage/RemoteGcsHelperTest.java | 23 +++++--- .../gcloud/storage/it/ITStorageTest.java | 55 ++++++++++++++----- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java index 024aa04eba1b..1287ede746d5 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java @@ -20,6 +20,7 @@ import com.google.gcloud.RetryParams; import com.google.gcloud.storage.BlobInfo; import com.google.gcloud.storage.Storage; +import com.google.gcloud.storage.Storage.BlobListOption; import com.google.gcloud.storage.StorageException; import com.google.gcloud.storage.StorageOptions; @@ -173,8 +174,8 @@ public DeleteBucketTask(Storage storage, String bucket) { @Override public Boolean call() { while (true) { - for (BlobInfo info : storage.list(bucket).values()) { - storage.delete(bucket, info.name()); + for (BlobInfo info : storage.list(bucket, BlobListOption.versions(true)).values()) { + storage.delete(info.blobId()); } try { storage.delete(bucket); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java index 2f56bbda7bd9..154554a029fe 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.gcloud.Page; +import com.google.gcloud.storage.Storage.BlobListOption; import com.google.gcloud.storage.testing.RemoteGcsHelper; import org.easymock.EasyMock; @@ -117,9 +118,10 @@ public Iterator iterateAll() { @Test public void testForceDelete() throws InterruptedException, ExecutionException { Storage storageMock = EasyMock.createMock(Storage.class); - EasyMock.expect(storageMock.list(BUCKET_NAME)).andReturn(blobPage); + EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) + .andReturn(blobPage); for (BlobInfo info : blobList) { - EasyMock.expect(storageMock.delete(BUCKET_NAME, info.name())).andReturn(true); + EasyMock.expect(storageMock.delete(info.blobId())).andReturn(true); } EasyMock.expect(storageMock.delete(BUCKET_NAME)).andReturn(true); EasyMock.replay(storageMock); @@ -132,7 +134,7 @@ public void testForceDeleteTimeout() throws InterruptedException, ExecutionExcep Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(storageMock.list(BUCKET_NAME)).andReturn(blobPage).anyTimes(); for (BlobInfo info : blobList) { - EasyMock.expect(storageMock.delete(BUCKET_NAME, info.name())).andReturn(true).anyTimes(); + EasyMock.expect(storageMock.delete(info.blobId())).andReturn(true).anyTimes(); } EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(RETRYABLE_EXCEPTION).anyTimes(); EasyMock.replay(storageMock); @@ -143,9 +145,10 @@ public void testForceDeleteTimeout() throws InterruptedException, ExecutionExcep @Test public void testForceDeleteFail() throws InterruptedException, ExecutionException { Storage storageMock = EasyMock.createMock(Storage.class); - EasyMock.expect(storageMock.list(BUCKET_NAME)).andReturn(blobPage); + EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) + .andReturn(blobPage); for (BlobInfo info : blobList) { - EasyMock.expect(storageMock.delete(BUCKET_NAME, info.name())).andReturn(true); + EasyMock.expect(storageMock.delete(info.blobId())).andReturn(true); } EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(FATAL_EXCEPTION); EasyMock.replay(storageMock); @@ -160,9 +163,10 @@ public void testForceDeleteFail() throws InterruptedException, ExecutionExceptio @Test public void testForceDeleteNoTimeout() { Storage storageMock = EasyMock.createMock(Storage.class); - EasyMock.expect(storageMock.list(BUCKET_NAME)).andReturn(blobPage); + EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) + .andReturn(blobPage); for (BlobInfo info : blobList) { - EasyMock.expect(storageMock.delete(BUCKET_NAME, info.name())).andReturn(true); + EasyMock.expect(storageMock.delete(info.blobId())).andReturn(true); } EasyMock.expect(storageMock.delete(BUCKET_NAME)).andReturn(true); EasyMock.replay(storageMock); @@ -173,9 +177,10 @@ public void testForceDeleteNoTimeout() { @Test public void testForceDeleteNoTimeoutFail() { Storage storageMock = EasyMock.createMock(Storage.class); - EasyMock.expect(storageMock.list(BUCKET_NAME)).andReturn(blobPage); + EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) + .andReturn(blobPage); for (BlobInfo info : blobList) { - EasyMock.expect(storageMock.delete(BUCKET_NAME, info.name())).andReturn(true); + EasyMock.expect(storageMock.delete(info.blobId())).andReturn(true); } EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(FATAL_EXCEPTION); EasyMock.replay(storageMock); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java index 43c2cf6d372b..1adf48a6cf68 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java @@ -28,23 +28,14 @@ import com.google.api.client.util.Lists; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.gcloud.Page; import com.google.gcloud.ReadChannel; import com.google.gcloud.RestorableState; import com.google.gcloud.WriteChannel; -import com.google.gcloud.storage.BatchRequest; -import com.google.gcloud.storage.BatchResponse; -import com.google.gcloud.storage.Blob; -import com.google.gcloud.storage.BlobId; -import com.google.gcloud.storage.BlobInfo; -import com.google.gcloud.storage.Bucket; -import com.google.gcloud.storage.BucketInfo; -import com.google.gcloud.storage.CopyWriter; -import com.google.gcloud.storage.HttpMethod; -import com.google.gcloud.storage.Storage; +import com.google.gcloud.storage.*; import com.google.gcloud.storage.Storage.BlobField; import com.google.gcloud.storage.Storage.BucketField; -import com.google.gcloud.storage.StorageException; import com.google.gcloud.storage.testing.RemoteGcsHelper; import org.junit.AfterClass; @@ -63,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -302,8 +294,7 @@ public void testListBlobsSelectedFields() { Blob remoteBlob2 = storage.create(blob2); assertNotNull(remoteBlob1); assertNotNull(remoteBlob2); - Page page = - storage.list(BUCKET, + Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), Storage.BlobListOption.fields(BlobField.METADATA)); int index = 0; @@ -331,8 +322,7 @@ public void testListBlobsEmptySelectedFields() { Blob remoteBlob2 = storage.create(blob2); assertNotNull(remoteBlob1); assertNotNull(remoteBlob2); - Page page = storage.list( - BUCKET, + Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), Storage.BlobListOption.fields()); int index = 0; @@ -345,6 +335,41 @@ public void testListBlobsEmptySelectedFields() { assertTrue(remoteBlob2.delete()); } + @Test + public void testListBlobsVersioned() throws ExecutionException, InterruptedException { + String bucketName = RemoteGcsHelper.generateBucketName(); + Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build()); + try { + String[] blobNames = {"test-list-blobs-versioned-blob1", "test-list-blobs-versioned-blob2"}; + BlobInfo blob1 = BlobInfo.builder(bucket, blobNames[0]) + .contentType(CONTENT_TYPE) + .build(); + BlobInfo blob2 = BlobInfo.builder(bucket, blobNames[1]) + .contentType(CONTENT_TYPE) + .build(); + Blob remoteBlob1 = storage.create(blob1); + Blob remoteBlob2 = storage.create(blob2); + Blob remoteBlob3 = storage.create(blob2); + assertNotNull(remoteBlob1); + assertNotNull(remoteBlob2); + assertNotNull(remoteBlob3); + Page page = storage.list(bucketName, + Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"), + Storage.BlobListOption.versions(true)); + Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); + for (Blob remoteBlob : page.values()) { + assertEquals(bucketName, remoteBlob.bucket()); + assertTrue(blobSet.contains(remoteBlob.name())); + assertNotNull(remoteBlob.generation()); + } + assertTrue(remoteBlob1.delete()); + assertTrue(remoteBlob2.delete()); + assertTrue(remoteBlob3.delete()); + } finally { + RemoteGcsHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + @Test public void testUpdateBlob() { String blobName = "test-update-blob"; From e66c9a48fc3dc6e952922d01375a2943161f7fca Mon Sep 17 00:00:00 2001 From: Arie Ozarov Date: Tue, 1 Mar 2016 07:13:29 -0800 Subject: [PATCH 3/3] fix style and formatting issue --- .../main/java/com/google/gcloud/storage/Storage.java | 3 +-- .../com/google/gcloud/storage/it/ITStorageTest.java | 12 +++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 4fa4ba3658be..98f3450b7f10 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -701,8 +701,7 @@ public static BlobListOption recursive(boolean recursive) { } /** - * If set to {@code true}, lists all versions of a blob. - * The default is {@code false}. + * If set to {@code true}, lists all versions of a blob. The default is {@code false}. * * @see Object Versioning */ diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java index 1adf48a6cf68..d239e40246d8 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java @@ -33,9 +33,19 @@ import com.google.gcloud.ReadChannel; import com.google.gcloud.RestorableState; import com.google.gcloud.WriteChannel; -import com.google.gcloud.storage.*; +import com.google.gcloud.storage.BatchRequest; +import com.google.gcloud.storage.BatchResponse; +import com.google.gcloud.storage.Blob; +import com.google.gcloud.storage.BlobId; +import com.google.gcloud.storage.BlobInfo; +import com.google.gcloud.storage.Bucket; +import com.google.gcloud.storage.BucketInfo; +import com.google.gcloud.storage.CopyWriter; +import com.google.gcloud.storage.HttpMethod; +import com.google.gcloud.storage.Storage; import com.google.gcloud.storage.Storage.BlobField; import com.google.gcloud.storage.Storage.BucketField; +import com.google.gcloud.storage.StorageException; import com.google.gcloud.storage.testing.RemoteGcsHelper; import org.junit.AfterClass;