diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fecac99..744523d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ on: env: GH_USER_NAME: github.actor SCRIPTS_VERSION: 5.7.0 - BOM_VERSION: 5.7.3 + BOM_VERSION: 5.7.4 REPOSITORY_URL: 'https://maven.pkg.github.com/' jobs: diff --git a/Dockerfile b/Dockerfile index a2e5cea..ea4124f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM alpine:latest -LABEL version=5.7.3 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu , Hleb Kanonik " +LABEL version=5.7.4 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu , Hleb Kanonik " ARG GH_TOKEN RUN apk -U -q upgrade && apk --no-cache -q add openjdk11 ca-certificates && \ - echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.3-exec.jar' > /start.sh && chmod +x /start.sh && \ - wget --header="Authorization: Bearer ${GH_TOKEN}" -q https://maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.3/service-jobs-5.7.3-exec.jar + echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.7.4-exec.jar' > /start.sh && chmod +x /start.sh && \ + wget --header="Authorization: Bearer ${GH_TOKEN}" -q https://maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.7.4/service-jobs-5.7.4-exec.jar ENV JAVA_OPTS="-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" VOLUME ["/tmp"] EXPOSE 8080 diff --git a/build.gradle b/build.gradle index dc2986f..8ef9b9f 100644 --- a/build.gradle +++ b/build.gradle @@ -72,7 +72,9 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-amqp' - implementation 'io.minio:minio:6.0.13' + implementation 'org.apache.jclouds.api:s3:2.5.0' + implementation 'org.apache.jclouds.provider:aws-s3:2.5.0' + implementation 'org.apache.httpcomponents:httpclient:4.5.13' // https://avd.aquasec.com/nvd/cve-2020-8908 // implementation 'com.google.guava:guava:30.0-jre'; diff --git a/gradle.properties b/gradle.properties index f947c21..8a54ff2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=5.7.4 +version=5.7.5 description=EPAM Report portal. Service jobs dockerServerUrl=unix:///var/run/docker.sock dockerPrepareEnvironment=apk -U -q upgrade && apk --no-cache -q add openjdk11 ca-certificates diff --git a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java index 58802e5..74e15ef 100644 --- a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java +++ b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java @@ -2,10 +2,20 @@ import com.epam.reportportal.storage.DataStorageService; import com.epam.reportportal.storage.LocalDataStorageService; -import com.epam.reportportal.storage.MinioDataStorageService; -import io.minio.MinioClient; -import io.minio.errors.InvalidEndpointException; -import io.minio.errors.InvalidPortException; +import com.epam.reportportal.storage.S3DataStorageService; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.cache.CacheLoader; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.inject.Module; +import org.jclouds.ContextBuilder; +import org.jclouds.aws.s3.config.AWSS3HttpApiModule; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.ContainerNotFoundException; +import org.jclouds.rest.ConfiguresHttpApi; +import org.jclouds.s3.S3Client; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -13,29 +23,137 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import java.util.Set; + @Configuration public class DataStorageConfig { - @Bean - @ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem") - public DataStorageService localDataStore(@Value("${datastore.default.path:/data/store}") String storagePath) { - return new LocalDataStorageService(storagePath); - } - - @Bean - @ConditionalOnProperty(name = "datastore.type", havingValue = "minio") - public MinioClient minioClient(@Value("${datastore.minio.endpoint}") String endpoint, - @Value("${datastore.minio.accessKey}") String accessKey, @Value("${datastore.minio.secretKey}") String secretKey, - @Value("${datastore.minio.region}") String region) throws InvalidPortException, InvalidEndpointException { - return new MinioClient(endpoint, accessKey, secretKey, region); - } - - @Bean - @Primary - @ConditionalOnProperty(name = "datastore.type", havingValue = "minio") - public DataStorageService minioDataStore(@Autowired MinioClient minioClient, - @Value("${datastore.minio.bucketPrefix}") String bucketPrefix, - @Value("${datastore.minio.defaultBucketName}") String defaultBucketName) { - return new MinioDataStorageService(minioClient, bucketPrefix, defaultBucketName); - } + /** + * Amazon has a general work flow they publish that allows clients to always find the correct URL endpoint for a given bucket: + * 1) ask s3.amazonaws.com for the bucket location + * 2) use the url returned to make the container specific request (get/put, etc) + * Jclouds cache the results from the first getBucketLocation call and use that region-specific URL, as needed. + * In this custom implementation of {@link AWSS3HttpApiModule} we are providing location from environment variable, so that + * we don't need to make getBucketLocation call + */ + @ConfiguresHttpApi + private static class CustomBucketToRegionModule extends AWSS3HttpApiModule { + private final String region; + + public CustomBucketToRegionModule(String region) { + this.region = region; + } + + @Override + @SuppressWarnings("Guava") + protected CacheLoader> bucketToRegion(Supplier> regionSupplier, S3Client client) { + Set regions = regionSupplier.get(); + if (regions.isEmpty()) { + return new CacheLoader<>() { + + @Override + @SuppressWarnings({ "Guava", "NullableProblems" }) + public Optional load(String bucket) { + if (CustomBucketToRegionModule.this.region != null) { + return Optional.of(CustomBucketToRegionModule.this.region); + } + return Optional.absent(); + } + + @Override + public String toString() { + return "noRegions()"; + } + }; + } else if (regions.size() == 1) { + final String onlyRegion = Iterables.getOnlyElement(regions); + return new CacheLoader<>() { + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + final Optional onlyRegionOption = Optional.of(onlyRegion); + + @Override + @SuppressWarnings("NullableProblems") + public Optional load(String bucket) { + if (CustomBucketToRegionModule.this.region != null) { + return Optional.of(CustomBucketToRegionModule.this.region); + } + return onlyRegionOption; + } + + @Override + public String toString() { + return "onlyRegion(" + onlyRegion + ")"; + } + }; + } else { + return new CacheLoader<>() { + @Override + @SuppressWarnings("NullableProblems") + public Optional load(String bucket) { + if (CustomBucketToRegionModule.this.region != null) { + return Optional.of(CustomBucketToRegionModule.this.region); + } + try { + return Optional.fromNullable(client.getBucketLocation(bucket)); + } catch (ContainerNotFoundException e) { + return Optional.absent(); + } + } + + @Override + public String toString() { + return "bucketToRegion()"; + } + }; + } + } + } + + @Bean + @ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem") + public DataStorageService localDataStore(@Value("${datastore.default.path:/data/store}") String storagePath) { + return new LocalDataStorageService(storagePath); + } + + @Bean + @ConditionalOnProperty(name = "datastore.type", havingValue = "minio") + public BlobStore minioBlobStore(@Value("${datastore.minio.accessKey}") String accessKey, + @Value("${datastore.minio.secretKey}") String secretKey, @Value("${datastore.minio.endpoint}") String endpoint) { + + BlobStoreContext blobStoreContext = ContextBuilder.newBuilder("s3") + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .buildView(BlobStoreContext.class); + + return blobStoreContext.getBlobStore(); + } + + @Bean + @ConditionalOnProperty(name = "datastore.type", havingValue = "minio") + public DataStorageService minioDataStore(@Autowired BlobStore blobStore, @Value("${datastore.minio.bucketPrefix}") String bucketPrefix, + @Value("${datastore.minio.defaultBucketName}") String defaultBucketName) { + return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName); + } + + @Bean + @ConditionalOnProperty(name = "datastore.type", havingValue = "s3") + public BlobStore blobStore(@Value("${datastore.s3.accessKey}") String accessKey, @Value("${datastore.s3.secretKey}") String secretKey, + @Value("${datastore.s3.region}") String region) { + Iterable modules = ImmutableSet.of(new CustomBucketToRegionModule(region)); + + BlobStoreContext blobStoreContext = ContextBuilder.newBuilder("aws-s3") + .modules(modules) + .credentials(accessKey, secretKey) + .buildView(BlobStoreContext.class); + + return blobStoreContext.getBlobStore(); + } + + @Bean + @Primary + @ConditionalOnProperty(name = "datastore.type", havingValue = "s3") + public DataStorageService s3DataStore(@Autowired BlobStore blobStore, @Value("${datastore.s3.bucketPrefix}") String bucketPrefix, + @Value("${datastore.s3.defaultBucketName}") String defaultBucketName) { + return new S3DataStorageService(blobStore, bucketPrefix, defaultBucketName); + } } diff --git a/src/main/java/com/epam/reportportal/storage/MinioDataStorageService.java b/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java similarity index 81% rename from src/main/java/com/epam/reportportal/storage/MinioDataStorageService.java rename to src/main/java/com/epam/reportportal/storage/S3DataStorageService.java index 5087689..1e910e8 100644 --- a/src/main/java/com/epam/reportportal/storage/MinioDataStorageService.java +++ b/src/main/java/com/epam/reportportal/storage/S3DataStorageService.java @@ -16,7 +16,7 @@ package com.epam.reportportal.storage; -import io.minio.MinioClient; +import org.jclouds.blobstore.BlobStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,18 +24,18 @@ import java.nio.file.Paths; /** - * Minio storage service + * S3 storage service */ -public class MinioDataStorageService implements DataStorageService { +public class S3DataStorageService implements DataStorageService { - private static final Logger LOGGER = LoggerFactory.getLogger(MinioDataStorageService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(S3DataStorageService.class); - private final MinioClient minioClient; + private final BlobStore blobStore; private final String bucketPrefix; private final String defaultBucketName; - public MinioDataStorageService(MinioClient minioClient, String bucketPrefix, String defaultBucketName) { - this.minioClient = minioClient; + public S3DataStorageService(BlobStore blobStore, String bucketPrefix, String defaultBucketName) { + this.blobStore = blobStore; this.bucketPrefix = bucketPrefix; this.defaultBucketName = defaultBucketName; } @@ -57,7 +57,7 @@ public void delete(String filePath) throws Exception { } try { - minioClient.removeObject(bucket, objectName); + blobStore.removeBlob(bucket, objectName); } catch (Exception e) { LOGGER.error("Unable to delete file '{}'", filePath, e); throw e; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 54a2009..b1550a3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -83,7 +83,7 @@ datastore: bucketPrefix: prj- defaultBucketName: rp-bucket region: #{null} - # could be one of [filesystem, minio] + # could be one of [filesystem, s3, minio] type: minio