Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support enumeration of filesystems when credentials are not available #1189

Merged
merged 7 commits into from
Aug 25, 2016
Merged
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -85,10 +85,11 @@
@AutoService(FileSystemProvider.class)
public final class CloudStorageFileSystemProvider extends FileSystemProvider {

private final Storage storage;
private Storage storage;
private StorageOptions storageOptions;

// used only when we create a new instance of CloudStorageFileSystemProvider.
private static StorageOptions storageOptions;
private static StorageOptions futureStorageOptions;

private static class LazyPathIterator extends AbstractIterator<Path> {
private final Iterator<Blob> blobIterator;
@@ -122,8 +123,8 @@ protected Path computeNext() {
* Sets options that are only used by the constructor.
*/
@VisibleForTesting
public static void setGCloudOptions(StorageOptions newStorageOptions) {
storageOptions = newStorageOptions;
public static void setStorageOptions(StorageOptions newStorageOptions) {
futureStorageOptions = newStorageOptions;

This comment was marked as spam.

This comment was marked as spam.

}

/**
@@ -133,14 +134,24 @@ public static void setGCloudOptions(StorageOptions newStorageOptions) {
* @see CloudStorageFileSystem#forBucket(String)
*/
public CloudStorageFileSystemProvider() {
this(storageOptions);
this(futureStorageOptions);
}

CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) {
if (gcsStorageOptions == null) {
this.storageOptions = gcsStorageOptions;

}

// Initialize this.storage, once. This may throw an exception if default authentication
// credentials are not available (hence not doing it in the ctor).
private void initStorage() {
if (this.storage != null) {
return;
}
if (storageOptions == null) {
this.storage = StorageOptions.defaultInstance().service();
} else {
this.storage = gcsStorageOptions.service();
this.storage = storageOptions.service();
}
}

@@ -154,6 +165,7 @@ public String getScheme() {
*/
@Override
public CloudStorageFileSystem getFileSystem(URI uri) {
initStorage();
return newFileSystem(uri, Collections.<String, Object>emptyMap());
}

@@ -186,11 +198,13 @@ && isNullOrEmpty(uri.getUserInfo()),
"GCS FileSystem URIs mustn't have: port, userinfo, path, query, or fragment: %s",
uri);
CloudStorageUtil.checkBucket(uri.getHost());
initStorage();
return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env));
}

@Override
public CloudStoragePath getPath(URI uri) {
initStorage();
return CloudStoragePath.getPath(
getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
}
@@ -199,6 +213,7 @@ public CloudStoragePath getPath(URI uri) {
public SeekableByteChannel newByteChannel(
Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
checkNotNull(path);
initStorage();
CloudStorageUtil.checkNotNullArray(attrs);
if (options.contains(StandardOpenOption.WRITE)) {
// TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option.
@@ -210,6 +225,7 @@ public SeekableByteChannel newByteChannel(

private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption> options)
throws IOException {
initStorage();
for (OpenOption option : options) {
if (option instanceof StandardOpenOption) {
switch ((StandardOpenOption) option) {
@@ -244,7 +260,7 @@ private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption>

private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption> options)
throws IOException {

initStorage();
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -318,6 +334,7 @@ private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption>

@Override
public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
initStorage();
InputStream result = super.newInputStream(path, options);
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
int blockSize = cloudPath.getFileSystem().config().blockSize();
@@ -331,6 +348,7 @@ public InputStream newInputStream(Path path, OpenOption... options) throws IOExc

@Override
public boolean deleteIfExists(Path path) throws IOException {
initStorage();
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -340,6 +358,7 @@ public boolean deleteIfExists(Path path) throws IOException {

@Override
public void delete(Path path) throws IOException {
initStorage();
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
if (!deleteIfExists(cloudPath)) {
throw new NoSuchFileException(cloudPath.toString());
@@ -348,6 +367,7 @@ public void delete(Path path) throws IOException {

@Override
public void move(Path source, Path target, CopyOption... options) throws IOException {
initStorage();
for (CopyOption option : options) {
if (option == StandardCopyOption.ATOMIC_MOVE) {
throw new AtomicMoveNotSupportedException(
@@ -362,6 +382,7 @@ public void move(Path source, Path target, CopyOption... options) throws IOExcep

@Override
public void copy(Path source, Path target, CopyOption... options) throws IOException {
initStorage();
boolean wantCopyAttributes = false;
boolean wantReplaceExisting = false;
boolean setContentType = false;
@@ -492,6 +513,7 @@ public boolean isHidden(Path path) {

@Override
public void checkAccess(Path path, AccessMode... modes) throws IOException {
initStorage();
for (AccessMode mode : modes) {
switch (mode) {
case READ:
@@ -520,6 +542,7 @@ public <A extends BasicFileAttributes> A readAttributes(
if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) {
throw new UnsupportedOperationException(type.getSimpleName());
}
initStorage();
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
@SuppressWarnings("unchecked")
@@ -574,6 +597,7 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
checkNotNull(filter);
initStorage();
String prefix = cloudPath.toString();
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
@@ -621,7 +645,9 @@ public int hashCode() {

@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("storage", storage).toString();
return "CloudStorageFileSystemProvider";
//initStorage();
//return MoreObjects.toStringHelper(this).add("storage", storage).toString();

This comment was marked as spam.

This comment was marked as spam.

}

private IOException asIoException(StorageException oops) {
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ public class CloudStorageFileAttributeViewTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
path = Paths.get(URI.create("gs://red/water"));
}

Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ public class CloudStorageFileAttributesTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
path = Paths.get(URI.create("gs://bucket/randompath"));
dir = Paths.get(URI.create("gs://bucket/randompath/"));
}
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ public class CloudStorageFileSystemProviderTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());

This comment was marked as spam.

This comment was marked as spam.

}

@Test
Original file line number Diff line number Diff line change
@@ -56,7 +56,7 @@ public class CloudStorageFileSystemTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
}

@Test
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright 2016 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.storage.contrib.nio;

import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

This comment was marked as spam.

This comment was marked as spam.


import java.net.URI;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

/**
* Unit tests for {@link CloudStorageFileSystemProvider} late initialization.
*/
@RunWith(JUnit4.class)
public class CloudStorageLateInitializationTest {

This comment was marked as spam.

This comment was marked as spam.


@Rule public final ExpectedException thrown = ExpectedException.none();

private StorageOptions mockOptions;

@Before
public void before() {
mockOptions = mock(StorageOptions.class);
Storage mockStorage = mock(Storage.class);
when(mockOptions.service()).thenReturn(mockStorage);
CloudStorageFileSystemProvider.setStorageOptions(mockOptions);
}

@Test
public void ctorDoesNotCreateStorage() {
new CloudStorageFileSystemProvider();
verify(mockOptions, never()).service();
}

@Test
public void getPathCreatesStorageOnce() {
CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider();
provider.getPath(URI.create("gs://bucket1/wat"));
provider.getPath(URI.create("gs://bucket2/wat"));
verify(mockOptions, times(1)).service();
}

@Test
public void getFileSystemCreatesStorageOnce() {
CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider();
provider.getFileSystem(URI.create("gs://bucket1"));
provider.getFileSystem(URI.create("gs://bucket2"));
verify(mockOptions, times(1)).service();
}

This comment was marked as spam.

}
Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ public class CloudStorageOptionsTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
}

@Test
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ public class CloudStoragePathTest {

@Before
public void before() {
CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
}

@Test