moved GCS and S3 related code to options classes

redis · May 10, 2021 · 44383a2 · 44383a2
1 parent 4498d0d
commit 44383a2
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 56 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -1,5 +1,5 @@
 group                 = com.redislabs
-version               = 2.11.5-SNAPSHOT
+version               = 2.11.5
 sourceCompatibility   = 1.8
 targetCompatibility   = 1.8
 picocliVersion        = 4.6.1
diff --git a/riot-file/src/main/java/com/redislabs/riot/file/FileOptions.java b/riot-file/src/main/java/com/redislabs/riot/file/FileOptions.java
@@ -1,20 +1,8 @@
 package com.redislabs.riot.file;
 
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.ServiceOptions;
-import com.google.cloud.storage.StorageOptions;
 import lombok.Data;
 import org.springframework.batch.item.resource.StandardInputResource;
 import org.springframework.batch.item.resource.StandardOutputResource;
-import org.springframework.cloud.aws.core.io.s3.SimpleStorageProtocolResolver;
-import org.springframework.cloud.gcp.autoconfigure.storage.GcpStorageAutoConfiguration;
-import org.springframework.cloud.gcp.core.GcpScope;
-import org.springframework.cloud.gcp.core.UserAgentHeaderProvider;
-import org.springframework.cloud.gcp.storage.GoogleStorageResource;
-import org.springframework.core.io.DefaultResourceLoader;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.WritableResource;
@@ -23,11 +11,8 @@
 import picocli.CommandLine.ArgGroup;
 import picocli.CommandLine.Option;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.util.Base64;
 
 @Data
 public class FileOptions {
@@ -39,9 +24,9 @@ public class FileOptions {
     @Option(names = {"-z", "--gzip"}, description = "File is gzip compressed")
     private boolean gzip;
     @ArgGroup(exclusive = false, heading = "Amazon Simple Storage Service options%n")
-    private S3Options s3;
+    private S3Options s3 = new S3Options();
     @ArgGroup(exclusive = false, heading = "Google Cloud Storage options%n")
-    private GcsOptions gcs;
+    private GcsOptions gcs = new GcsOptions();
 
     @SuppressWarnings({"unchecked", "FieldCanBeLocal"})
     public static class FileOptionsBuilder<B extends FileOptionsBuilder<B>> {
@@ -106,54 +91,18 @@ public WritableResource outputResource(String file) throws IOException {
         return writable;
     }
 
-    public Resource s3Resource(String location) {
-        AmazonS3ClientBuilder clientBuilder = AmazonS3Client.builder();
-        if (s3 != null) {
-            if (s3.getRegion() != null) {
-                clientBuilder.withRegion(s3.getRegion());
-            }
-            if (s3.getAccessKey() != null) {
-                clientBuilder.withCredentials(new SimpleAWSCredentialsProvider(s3.getAccessKey(), s3.getSecretKey()));
-            }
-        }
-        SimpleStorageProtocolResolver resolver = new SimpleStorageProtocolResolver() {
-            @Override
-            public AmazonS3 getAmazonS3() {
-                return clientBuilder.build();
-            }
-        };
-        resolver.afterPropertiesSet();
-        return resolver.resolve(location, new DefaultResourceLoader());
-    }
-
     private Resource resource(String location, boolean readOnly) throws IOException {
         if (FileUtils.isS3(location)) {
-            return s3Resource(location);
+            return s3.resource(location);
         }
         if (FileUtils.isGcs(location)) {
-            return gcsResource(location, readOnly);
+            return gcs.resource(location, readOnly);
         }
         if (ResourceUtils.isUrl(location)) {
             return new UncustomizedUrlResource(location);
         }
         return new FileSystemResource(location);
     }
 
-    private GoogleStorageResource gcsResource(String locationUri, boolean readOnly) throws IOException {
-        StorageOptions.Builder builder = StorageOptions.newBuilder().setProjectId(ServiceOptions.getDefaultProjectId()).setHeaderProvider(new UserAgentHeaderProvider(GcpStorageAutoConfiguration.class));
-        if (gcs != null) {
-            if (gcs.getCredentials() != null) {
-                builder.setCredentials(GoogleCredentials.fromStream(Files.newInputStream(gcs.getCredentials().toPath())).createScoped((readOnly ? GcpScope.STORAGE_READ_ONLY : GcpScope.STORAGE_READ_WRITE).getUrl()));
-            }
-            if (gcs.getEncodedKey() != null) {
-                builder.setCredentials(GoogleCredentials.fromStream(new ByteArrayInputStream(Base64.getDecoder().decode(gcs.getEncodedKey()))));
-            }
-            if (gcs.getProjectId() != null) {
-                builder.setProjectId(gcs.getProjectId());
-            }
-        }
-        return new GoogleStorageResource(builder.build().getService(), locationUri);
-    }
-
 }
 
diff --git a/riot-file/src/main/java/com/redislabs/riot/file/GcsOptions.java b/riot-file/src/main/java/com/redislabs/riot/file/GcsOptions.java
@@ -1,9 +1,20 @@
 package com.redislabs.riot.file;
 
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.storage.StorageOptions;
 import lombok.Data;
+import org.springframework.cloud.gcp.autoconfigure.storage.GcpStorageAutoConfiguration;
+import org.springframework.cloud.gcp.core.GcpScope;
+import org.springframework.cloud.gcp.core.UserAgentHeaderProvider;
+import org.springframework.cloud.gcp.storage.GoogleStorageResource;
 import picocli.CommandLine.Option;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Base64;
 
 @Data
 public class GcsOptions {
@@ -15,4 +26,18 @@ public class GcsOptions {
     @Option(names = "--gcs-key", arity = "0..1", interactive = true, description = "GCS Base64 encoded key", paramLabel = "<key>")
     private String encodedKey;
 
+    public GoogleStorageResource resource(String locationUri, boolean readOnly) throws IOException {
+        StorageOptions.Builder builder = StorageOptions.newBuilder().setProjectId(ServiceOptions.getDefaultProjectId()).setHeaderProvider(new UserAgentHeaderProvider(GcpStorageAutoConfiguration.class));
+        if (credentials != null) {
+            builder.setCredentials(GoogleCredentials.fromStream(Files.newInputStream(credentials.toPath())).createScoped((readOnly ? GcpScope.STORAGE_READ_ONLY : GcpScope.STORAGE_READ_WRITE).getUrl()));
+        }
+        if (encodedKey != null) {
+            builder.setCredentials(GoogleCredentials.fromStream(new ByteArrayInputStream(Base64.getDecoder().decode(encodedKey))));
+        }
+        if (projectId != null) {
+            builder.setProjectId(projectId);
+        }
+        return new GoogleStorageResource(builder.build().getService(), locationUri);
+    }
+
 }
diff --git a/riot-file/src/main/java/com/redislabs/riot/file/S3Options.java b/riot-file/src/main/java/com/redislabs/riot/file/S3Options.java
@@ -1,6 +1,12 @@
 package com.redislabs.riot.file;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import lombok.Data;
+import org.springframework.cloud.aws.core.io.s3.SimpleStorageProtocolResolver;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
 import picocli.CommandLine.Option;
 
 @Data
@@ -13,4 +19,21 @@ public class S3Options {
     @Option(names = "--s3-region", description = "AWS region", paramLabel = "<name>")
     private String region;
 
+    public Resource resource(String location) {
+        AmazonS3ClientBuilder clientBuilder = AmazonS3Client.builder();
+        if (region != null) {
+            clientBuilder.withRegion(region);
+        }
+        if (accessKey != null) {
+            clientBuilder.withCredentials(new SimpleAWSCredentialsProvider(accessKey, secretKey));
+        }
+        SimpleStorageProtocolResolver resolver = new SimpleStorageProtocolResolver() {
+            @Override
+            public AmazonS3 getAmazonS3() {
+                return clientBuilder.build();
+            }
+        };
+        resolver.afterPropertiesSet();
+        return resolver.resolve(location, new DefaultResourceLoader());
+    }
 }