Merge pull request #884 from jean-philippe-martin/fake_list

Implement bare-bones blob listing

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java

@@ -29,8 +29,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Throwables;
+import com.google.common.collect.AbstractIterator;
 import com.google.common.primitives.Ints;
 import com.google.gcloud.storage.Acl;
+import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.BlobId;
 import com.google.gcloud.storage.BlobInfo;
 import com.google.gcloud.storage.CopyWriter;
@@ -46,6 +48,7 @@
 import java.nio.file.AccessMode;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.CopyOption;
+import java.nio.file.DirectoryIteratorException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.DirectoryStream.Filter;
 import java.nio.file.FileAlreadyExistsException;
@@ -64,11 +67,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
 
@@ -84,6 +87,33 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider {
   // used only when we create a new instance of CloudStorageFileSystemProvider.
   private static StorageOptions defaultStorageOptions;
 
+  private static class LazyPathIterator extends AbstractIterator<Path> {
+    private final Iterator<Blob> blobIterator;
+    private final Filter<? super Path> filter;
+    private final CloudStorageFileSystem fileSystem;
+
+    LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
+      this.blobIterator = blobIterator;
+      this.filter = filter;
+      this.fileSystem = fileSystem;
+    }
+
+    @Override
+    protected Path computeNext() {
+      while (blobIterator.hasNext()) {
+        Path path = fileSystem.getPath(blobIterator.next().name());
+        try {
+          if (filter.accept(path)) {
+            return path;
+          }
+        } catch (IOException ex) {
+          throw new DirectoryIteratorException(ex);
+        }
+      }
+      return endOfData();
+    }
+  }
+
   /**
    * Sets default options that are only used by the constructor.
    */
@@ -532,13 +562,23 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
     checkNotNullArray(attrs);
   }
 
-  /**
-   * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet.
-   */
   @Override
-  public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) {
-    // TODO: Implement me.
-    throw new UnsupportedOperationException();
+  public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
+    final CloudStoragePath cloudPath = checkPath(dir);
+    checkNotNull(filter);
+    String prefix = cloudPath.toString();
+    final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
+    return new DirectoryStream<Path>() {
+      @Override
+      public Iterator<Path> iterator() {
+        return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter);
+      }
+
+      @Override
+      public void close() throws IOException {
+        // Does nothing since there's nothing to close. Commenting this method to quiet codacy.
+      }
+    };
   }
 
   /**

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java

@@ -34,7 +34,10 @@
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Unit tests for {@link CloudStorageFileSystem}.
@@ -134,4 +137,27 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx
       tester.testAllPublicInstanceMethods(fs);
     }
   }
+
+  @Test
+  public void testListFiles() throws IOException {
+    try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {
+      List<Path> goodPaths = new ArrayList<>();
+      List<Path> paths = new ArrayList<>();
+      goodPaths.add(fs.getPath("dir/angel"));
+      goodPaths.add(fs.getPath("dir/alone"));
+      paths.add(fs.getPath("dir/dir2/another_angel"));
+      paths.add(fs.getPath("atroot"));
+      paths.addAll(goodPaths);
+      goodPaths.add(fs.getPath("dir/dir2/"));
+      for (Path path : paths) {
+        Files.write(path, ALONE.getBytes(UTF_8));
+      }
+
+      List<Path> got = new ArrayList<>();
+      for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) {
+        got.add(path);
+      }
+      assertThat(got).containsExactlyElementsIn(goodPaths);
+    }
+  }
 }

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java

@@ -22,14 +22,12 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 import com.google.gcloud.ReadChannel;
 import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.BlobId;
-import com.google.gcloud.storage.BlobInfo;
 import com.google.gcloud.storage.Storage;
 
 import org.junit.Before;

gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java

@@ -26,8 +26,10 @@
 import java.io.InputStream;
 import java.math.BigInteger;
 import java.nio.file.FileAlreadyExistsException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import javax.annotation.concurrent.NotThreadSafe;
@@ -91,8 +93,49 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
   @Override
   public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
       throws StorageException {
-    potentiallyThrow(options);
-    return null;
+    String preprefix = "";
+    for (Map.Entry<Option, ?> e : options.entrySet()) {
+      switch (e.getKey()) {
+        case PREFIX:
+          preprefix = (String) e.getValue();
+          if (preprefix.startsWith("/")) {
+            preprefix = preprefix.substring(1);
+          }
+          break;
+        case FIELDS:
+          // ignore and return all the fields
+          break;
+        default:
+          throw new UnsupportedOperationException("Unknown option: " + e.getKey());
+      }
+    }
+    final String prefix = preprefix;
+
+    List<StorageObject> values = new ArrayList<>();
+    Map<String, StorageObject> folders = new HashMap<>();
+    for (StorageObject so : stuff.values()) {
+      if (!so.getName().startsWith(prefix)) {
+        continue;
+      }
+      int nextSlash = so.getName().indexOf("/", prefix.length());
+      if (nextSlash >= 0) {
+        String folderName = so.getName().substring(0, nextSlash + 1);
+        if (folders.containsKey(folderName)) {
+          continue;
+        }
+        StorageObject fakeFolder = new StorageObject();
+        fakeFolder.setName(folderName);
+        fakeFolder.setBucket(so.getBucket());
+        fakeFolder.setGeneration(so.getGeneration());
+        folders.put(folderName, fakeFolder);
+        continue;
+      }
+      values.add(so);
+    }
+    values.addAll(folders.values());
+    // null cursor to indicate there is no more data (empty string would cause us to be called again).
+    // The type cast seems to be necessary to help Java's typesystem remember that collections are iterable.
+    return Tuple.of(null, (Iterable<StorageObject>) values);
   }
 
   /**
@@ -111,7 +154,7 @@ public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException
   public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
     // we allow the "ID" option because we need to, but then we give a whole answer anyways
     // because the caller won't mind the extra fields.
-    if (throwIfOption && !options.isEmpty() && options.size()>1
+    if (throwIfOption && !options.isEmpty() && options.size() > 1
         && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
       throw new UnsupportedOperationException();
     }