EPMRPP-80865 || Merge master to develop

.github/workflows/release.yml

@@ -12,7 +12,7 @@ on:
 env:
   GH_USER_NAME: github.actor
   SCRIPTS_VERSION: 5.7.0
-  BOM_VERSION: 5.7.3
+  BOM_VERSION: 5.7.4
   REPOSITORY_URL: 'https://maven.pkg.github.com/'
 
 jobs:

Dockerfile

@@ -1,9 +1,9 @@
 FROM alpine:latest
-LABEL version=5.7.3 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu <[email protected]>, Hleb Kanonik <[email protected]>"
+LABEL version=5.7.4 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu <[email protected]>, Hleb Kanonik <[email protected]>"
 ARG GH_TOKEN
 RUN apk -U -q upgrade && apk --no-cache -q add openjdk11 ca-certificates && \
-	echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.3-exec.jar' > /start.sh && chmod +x /start.sh && \
-	wget --header="Authorization: Bearer ${GH_TOKEN}"  -q https://maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.3/service-jobs-5.7.3-exec.jar
+	echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.4-exec.jar' > /start.sh && chmod +x /start.sh && \
+	wget --header="Authorization: Bearer ${GH_TOKEN}"  -q https://maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.4/service-jobs-5.7.4-exec.jar
 ENV JAVA_OPTS="-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom"
 VOLUME ["/tmp"]
 EXPOSE 8080

build.gradle

@@ -72,7 +72,9 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-jdbc'
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'org.springframework.boot:spring-boot-starter-amqp'
-    implementation 'io.minio:minio:6.0.13'
+    implementation 'org.apache.jclouds.api:s3:2.5.0'
+    implementation 'org.apache.jclouds.provider:aws-s3:2.5.0'
+    implementation 'org.apache.httpcomponents:httpclient:4.5.13'
 //    https://avd.aquasec.com/nvd/cve-2020-8908
 //    implementation 'com.google.guava:guava:30.0-jre';
 

gradle.properties

@@ -1,4 +1,4 @@
-version=5.7.4
+version=5.7.5
 description=EPAM Report portal. Service jobs
 dockerServerUrl=unix:///var/run/docker.sock
 dockerPrepareEnvironment=apk -U -q upgrade && apk --no-cache -q add openjdk11 ca-certificates

src/main/java/com/epam/reportportal/config/DataStorageConfig.java

@@ -2,40 +2,158 @@
 
 import com.epam.reportportal.storage.DataStorageService;
 import com.epam.reportportal.storage.LocalDataStorageService;
-import com.epam.reportportal.storage.MinioDataStorageService;
-import io.minio.MinioClient;
-import io.minio.errors.InvalidEndpointException;
-import io.minio.errors.InvalidPortException;
+import com.epam.reportportal.storage.S3DataStorageService;
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.cache.CacheLoader;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.inject.Module;
+import org.jclouds.ContextBuilder;
+import org.jclouds.aws.s3.config.AWSS3HttpApiModule;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.rest.ConfiguresHttpApi;
+import org.jclouds.s3.S3Client;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
 
+import java.util.Set;
+
 @Configuration
 public class DataStorageConfig {
 
-    @Bean
-    @ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem")
-    public DataStorageService localDataStore(@Value("${datastore.default.path:/data/store}") String storagePath) {
-        return new LocalDataStorageService(storagePath);
-    }
-
-    @Bean
-    @ConditionalOnProperty(name = "datastore.type", havingValue = "minio")
-    public MinioClient minioClient(@Value("${datastore.minio.endpoint}") String endpoint,
-                                   @Value("${datastore.minio.accessKey}") String accessKey, @Value("${datastore.minio.secretKey}") String secretKey,
-                                   @Value("${datastore.minio.region}") String region) throws InvalidPortException, InvalidEndpointException {
-        return new MinioClient(endpoint, accessKey, secretKey, region);
-    }
-
-    @Bean
-    @Primary
-    @ConditionalOnProperty(name = "datastore.type", havingValue = "minio")
-    public DataStorageService minioDataStore(@Autowired MinioClient minioClient,
-                                                  @Value("${datastore.minio.bucketPrefix}") String bucketPrefix,
-                                                  @Value("${datastore.minio.defaultBucketName}") String defaultBucketName) {
-        return new MinioDataStorageService(minioClient, bucketPrefix, defaultBucketName);
-    }
+	/**
+	 * Amazon has a general work flow they publish that allows clients to always find the correct URL endpoint for a given bucket:
+	 * 1) ask s3.amazonaws.com for the bucket location
+	 * 2) use the url returned to make the container specific request (get/put, etc)
+	 * Jclouds cache the results from the first getBucketLocation call and use that region-specific URL, as needed.
+	 * In this custom implementation of {@link AWSS3HttpApiModule} we are providing location from environment variable, so that
+	 * we don't need to make getBucketLocation call
+	 */
+	@ConfiguresHttpApi
+	private static class CustomBucketToRegionModule extends AWSS3HttpApiModule {
+		private final String region;
+
+		public CustomBucketToRegionModule(String region) {
+			this.region = region;
+		}
+
+		@Override
+		@SuppressWarnings("Guava")
+		protected CacheLoader<String, Optional<String>> bucketToRegion(Supplier<Set<String>> regionSupplier, S3Client client) {
+			Set<String> regions = regionSupplier.get();
+			if (regions.isEmpty()) {
+				return new CacheLoader<>() {
+
+					@Override
+					@SuppressWarnings({ "Guava", "NullableProblems" })
+					public Optional<String> load(String bucket) {
+						if (CustomBucketToRegionModule.this.region != null) {
+							return Optional.of(CustomBucketToRegionModule.this.region);
+						}
+						return Optional.absent();
+					}
+
+					@Override
+					public String toString() {
+						return "noRegions()";
+					}
+				};
+			} else if (regions.size() == 1) {
+				final String onlyRegion = Iterables.getOnlyElement(regions);
+				return new CacheLoader<>() {
+					@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
+					final Optional<String> onlyRegionOption = Optional.of(onlyRegion);
+
+					@Override
+					@SuppressWarnings("NullableProblems")
+					public Optional<String> load(String bucket) {
+						if (CustomBucketToRegionModule.this.region != null) {
+							return Optional.of(CustomBucketToRegionModule.this.region);
+						}
+						return onlyRegionOption;
+					}
+
+					@Override
+					public String toString() {
+						return "onlyRegion(" + onlyRegion + ")";
+					}
+				};
+			} else {
+				return new CacheLoader<>() {
+					@Override
+					@SuppressWarnings("NullableProblems")
+					public Optional<String> load(String bucket) {
+						if (CustomBucketToRegionModule.this.region != null) {
+							return Optional.of(CustomBucketToRegionModule.this.region);
+						}
+						try {
+							return Optional.fromNullable(client.getBucketLocation(bucket));
+						} catch (ContainerNotFoundException e) {
+							return Optional.absent();
+						}
+					}
+
+					@Override
+					public String toString() {
+						return "bucketToRegion()";
+					}
+				};
+			}
+		}
+	}
+
+	@Bean
+	@ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem")
+	public DataStorageService localDataStore(@Value("${datastore.default.path:/data/store}") String storagePath) {
+		return new LocalDataStorageService(storagePath);
+	}
+
+	@Bean
+	@ConditionalOnProperty(name = "datastore.type", havingValue = "minio")
+	public BlobStore minioBlobStore(@Value("${datastore.minio.accessKey}") String accessKey,
+			@Value("${datastore.minio.secretKey}") String secretKey, @Value("${datastore.minio.endpoint}") String endpoint) {
+
+		BlobStoreContext blobStoreContext = ContextBuilder.newBuilder("s3")
+				.endpoint(endpoint)
+				.credentials(accessKey, secretKey)
+				.buildView(BlobStoreContext.class);
+
+		return blobStoreContext.getBlobStore();
+	}
+
+	@Bean
+	@ConditionalOnProperty(name = "datastore.type", havingValue = "minio")
+	public DataStorageService minioDataStore(@Autowired BlobStore blobStore, @Value("${datastore.minio.bucketPrefix}") String bucketPrefix,
+			@Value("${datastore.minio.defaultBucketName}") String defaultBucketName) {
+		return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName);
+	}
+
+	@Bean
+	@ConditionalOnProperty(name = "datastore.type", havingValue = "s3")
+	public BlobStore blobStore(@Value("${datastore.s3.accessKey}") String accessKey, @Value("${datastore.s3.secretKey}") String secretKey,
+			@Value("${datastore.s3.region}") String region) {
+		Iterable<Module> modules = ImmutableSet.of(new CustomBucketToRegionModule(region));
+
+		BlobStoreContext blobStoreContext = ContextBuilder.newBuilder("aws-s3")
+				.modules(modules)
+				.credentials(accessKey, secretKey)
+				.buildView(BlobStoreContext.class);
+
+		return blobStoreContext.getBlobStore();
+	}
+
+	@Bean
+	@Primary
+	@ConditionalOnProperty(name = "datastore.type", havingValue = "s3")
+	public DataStorageService s3DataStore(@Autowired BlobStore blobStore, @Value("${datastore.s3.bucketPrefix}") String bucketPrefix,
+			@Value("${datastore.s3.defaultBucketName}") String defaultBucketName) {
+		return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName);
+	}
 }

src/main/java/com/epam/reportportal/storage/MinioDataStorageService.java → src/main/java/com/epam/reportportal/storage/S3DataStorageService.java

@@ -16,26 +16,26 @@
 
 package com.epam.reportportal.storage;
 
-import io.minio.MinioClient;
+import org.jclouds.blobstore.BlobStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 /**
- * Minio storage service
+ * S3 storage service
  */
-public class MinioDataStorageService implements DataStorageService {
+public class S3DataStorageService implements DataStorageService {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(MinioDataStorageService.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(S3DataStorageService.class);
 
-    private final MinioClient minioClient;
+    private final BlobStore blobStore;
     private final String bucketPrefix;
     private final String defaultBucketName;
 
-    public MinioDataStorageService(MinioClient minioClient, String bucketPrefix, String defaultBucketName) {
-        this.minioClient = minioClient;
+    public S3DataStorageService(BlobStore blobStore, String bucketPrefix, String defaultBucketName) {
+        this.blobStore = blobStore;
         this.bucketPrefix = bucketPrefix;
         this.defaultBucketName = defaultBucketName;
     }
@@ -57,7 +57,7 @@ public void delete(String filePath) throws Exception {
         }
 
         try {
-            minioClient.removeObject(bucket, objectName);
+            blobStore.removeBlob(bucket, objectName);
         } catch (Exception e) {
             LOGGER.error("Unable to delete file '{}'", filePath, e);
             throw e;

src/main/resources/application.yml

@@ -83,7 +83,7 @@ datastore:
     bucketPrefix: prj-
     defaultBucketName: rp-bucket
     region: #{null}
-  # could be one of [filesystem, minio]
+  # could be one of [filesystem, s3, minio]
   type: minio