Support enumeration of filesystems when credentials are not available

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java

@@ -85,10 +85,11 @@
 @AutoService(FileSystemProvider.class)
 public final class CloudStorageFileSystemProvider extends FileSystemProvider {
 
-  private final Storage storage;
+  private Storage storage;
+  private StorageOptions storageOptions;
 
   // used only when we create a new instance of CloudStorageFileSystemProvider.
-  private static StorageOptions storageOptions;
+  private static StorageOptions futureStorageOptions;
 
   private static class LazyPathIterator extends AbstractIterator<Path> {
     private final Iterator<Blob> blobIterator;
@@ -123,7 +124,7 @@ protected Path computeNext() {
    */
   @VisibleForTesting
   public static void setGCloudOptions(StorageOptions newStorageOptions) {
-    storageOptions = newStorageOptions;
+    futureStorageOptions = newStorageOptions;
   }
 
   /**
@@ -133,14 +134,26 @@ public static void setGCloudOptions(StorageOptions newStorageOptions) {
    * @see CloudStorageFileSystem#forBucket(String)
    */
   public CloudStorageFileSystemProvider() {
-    this(storageOptions);
+    this(futureStorageOptions);
   }
 
   CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) {
-    if (gcsStorageOptions == null) {
+    this.storageOptions = gcsStorageOptions;
+
+  }
+
+  // Initialize this.storage, once. This may throw an exception if the environment variable
+  // GOOGLE_APPLICATION_CREDENTIALS is not set.
+  // We don't do this in the constructor because the ctor is called even when just enumerating
+  // available filesystems, and this should work without configuring gcloud.
+  private void initStorage() {
+    if (this.storage != null) {
+      return;
+    }
+    if (storageOptions == null) {
       this.storage = StorageOptions.defaultInstance().service();
     } else {
-      this.storage = gcsStorageOptions.service();
+      this.storage = storageOptions.service();
     }
   }
 
@@ -154,6 +167,7 @@ public String getScheme() {
    */
   @Override
   public CloudStorageFileSystem getFileSystem(URI uri) {
+    initStorage();
     return newFileSystem(uri, Collections.<String, Object>emptyMap());
   }
 
@@ -186,11 +200,13 @@ && isNullOrEmpty(uri.getUserInfo()),
         "GCS FileSystem URIs mustn't have: port, userinfo, path, query, or fragment: %s",
         uri);
     CloudStorageUtil.checkBucket(uri.getHost());
+    initStorage();
     return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env));
   }
 
   @Override
   public CloudStoragePath getPath(URI uri) {
+    initStorage();
     return CloudStoragePath.getPath(
         getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
   }
@@ -199,6 +215,7 @@ public CloudStoragePath getPath(URI uri) {
   public SeekableByteChannel newByteChannel(
       Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
     checkNotNull(path);
+    initStorage();
     CloudStorageUtil.checkNotNullArray(attrs);
     if (options.contains(StandardOpenOption.WRITE)) {
       // TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option.
@@ -210,6 +227,7 @@ public SeekableByteChannel newByteChannel(
 
   private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption> options)
       throws IOException {
+    initStorage();
     for (OpenOption option : options) {
       if (option instanceof StandardOpenOption) {
         switch ((StandardOpenOption) option) {
@@ -244,7 +262,7 @@ private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption>
 
   private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption> options)
       throws IOException {
-
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -318,6 +336,7 @@ private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption>
 
   @Override
   public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
+    initStorage();
     InputStream result = super.newInputStream(path, options);
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     int blockSize = cloudPath.getFileSystem().config().blockSize();
@@ -331,6 +350,7 @@ public InputStream newInputStream(Path path, OpenOption... options) throws IOExc
 
   @Override
   public boolean deleteIfExists(Path path) throws IOException {
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -340,6 +360,7 @@ public boolean deleteIfExists(Path path) throws IOException {
 
   @Override
   public void delete(Path path) throws IOException {
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (!deleteIfExists(cloudPath)) {
       throw new NoSuchFileException(cloudPath.toString());
@@ -348,6 +369,7 @@ public void delete(Path path) throws IOException {
 
   @Override
   public void move(Path source, Path target, CopyOption... options) throws IOException {
+    initStorage();
     for (CopyOption option : options) {
       if (option == StandardCopyOption.ATOMIC_MOVE) {
         throw new AtomicMoveNotSupportedException(
@@ -362,6 +384,7 @@ public void move(Path source, Path target, CopyOption... options) throws IOExcep
 
   @Override
   public void copy(Path source, Path target, CopyOption... options) throws IOException {
+    initStorage();
     boolean wantCopyAttributes = false;
     boolean wantReplaceExisting = false;
     boolean setContentType = false;
@@ -492,6 +515,7 @@ public boolean isHidden(Path path) {
 
   @Override
   public void checkAccess(Path path, AccessMode... modes) throws IOException {
+    initStorage();
     for (AccessMode mode : modes) {
       switch (mode) {
         case READ:
@@ -520,6 +544,7 @@ public <A extends BasicFileAttributes> A readAttributes(
     if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) {
       throw new UnsupportedOperationException(type.getSimpleName());
     }
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       @SuppressWarnings("unchecked")
@@ -574,6 +599,7 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
   public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
     final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
     checkNotNull(filter);
+    initStorage();
     String prefix = cloudPath.toString();
     final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
         Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
@@ -621,7 +647,9 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper(this).add("storage", storage).toString();
+    return "CloudStorageFileSystemProvider";
+    //initStorage();
+    //return MoreObjects.toStringHelper(this).add("storage", storage).toString();
   }
 
   private IOException asIoException(StorageException oops) {

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageLateInitializationTest.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage.contrib.nio;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.testing.NullPointerTester;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.CopyOption;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.List;
+
+import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.forBucket;
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.CREATE_NEW;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ * Unit tests for {@link CloudStorageFileSystemProvider} when gcloud is not configured.
+ */
+//@RunWith(JUnit4.class)
+@Ignore // these tests must be run manually because they require gcloud to NOT be configured.
+public class CloudStorageLateInitializationTest {
+
+  @Rule public final ExpectedException thrown = ExpectedException.none();
+
+  @Before
+  public void before() {
+    if (System.getenv().containsKey("GOOGLE_APPLICATION_CREDENTIALS")) {
+      throw new RuntimeException("CloudStorageLateInitializationTest can only be run if gcloud is not configured. This means no GOOGLE_APPLICATION_CREDENTIALS environment variable.");
+    }
+    // actually that one's OK so long as it doesn't point to a valid configuration, but it's simpler
+    // to check that it isn't there.
+    if (System.getenv().containsKey("CLOUDSDK_CONFIG")) {
+      throw new RuntimeException("CloudStorageLateInitializationTest can only be run if gcloud is" +
+          " not configured. This means no CLOUDSDK_CONFIG environment variable.");
+    }
+    if (Files.exists(Paths.get(System.getenv("HOME"), ".config/gcloud/properties"))) {
+      throw new RuntimeException("CloudStorageLateInitializationTest can only be run if gcloud is" +
+          " not configured. This means no ~/.config/gcloud/properties.");
+    }
+    if (Files.exists(Paths.get(System.getenv("HOME"), ".config/gcloud/active_config"))) {
+      throw new RuntimeException("CloudStorageLateInitializationTest can only be run if gcloud is" +
+          " not configured. This means no ~/.config/gcloud/active_config.");
+    }
+  }
+
+  @Test(expected = java.lang.IllegalArgumentException.class)
+  public void pathFailsIfNoEnvVariable() throws IOException {
+    // this should fail if we haven't set GOOGLE_APPLICATION_CREDENTIALS nor CLOUDSDK_CONFIG
+    // *and* we don't have a ~/.config/gcloud/properties file (or %APPDATA%/gcloud in Windows)
+    // and we don't have a ~/.config/gcloud/active-config.
+    // (since we're also not providing credentials in any other way)
+    Path path = Paths.get(URI.create("gs://bucket/wat"));
+  }
+
+  @Test
+  public void enumerateFilesystemsIfNoEnvVariable() throws IOException {
+    // listing available filesystem providers should work even if we can't initialize
+    // CloudStorageFilesystemProvider. This makes it possible to use other filesystems
+    // when gcloud-java is in the classpath but gcloud isn't configured.
+    System.out.println("Installed filesystem providers:");
+    for (FileSystemProvider p : FileSystemProvider.installedProviders()) {
+      System.out.println("  " + p.getScheme());
+    }
+  }
+
+}

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java

@@ -96,6 +96,11 @@ public class ITGcsNio {
 
   @BeforeClass
   public static void beforeClass() throws IOException {
+    if (!System.getenv().containsKey("GOOGLE_APPLICATION_CREDENTIALS")) {
+      throw new RuntimeException("ITGcsNio can only be run if gcloud is configured. Please set " +
+          " the GOOGLE_APPLICATION_CREDENTIALS environment variable.");
+    }
+
     // loads the credentials from local disk as par README
     RemoteStorageHelper gcsHelper = RemoteStorageHelper.create();
     storageOptions = gcsHelper.options();