diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java index a10d9515..97644a88 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java @@ -18,7 +18,7 @@ import com.ionos.edc.extension.s3.connector.ionosapi.TemporaryKey; -import java.io.InputStream; +import java.io.ByteArrayInputStream; import java.util.List; @ExtensionPoint @@ -28,9 +28,11 @@ public interface S3ConnectorApi { boolean bucketExists(String bucketName); - void uploadObject(String bucketName, String objectName, InputStream stream); + void uploadObject(String bucketName, String objectName, ByteArrayInputStream stream); - InputStream getObject(String bucketName, String objectName); + ByteArrayInputStream getObject(String bucketName, String objectName); + + ByteArrayInputStream getObject(String bucketName, String objectName, long offset, long length); List listObjects(String bucketName, String objectName); diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java index daf08dcc..5591583a 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java @@ -18,17 +18,22 @@ import com.ionos.edc.extension.s3.connector.ionosapi.HttpConnector; import com.ionos.edc.extension.s3.connector.ionosapi.TemporaryKey; -import io.minio.*; +import io.minio.BucketExistsArgs; +import io.minio.GetObjectArgs; +import io.minio.ListObjectsArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; import org.eclipse.edc.spi.EdcException; -import java.io.InputStream; +import java.io.ByteArrayInputStream; import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; public class S3ConnectorApiImpl implements S3ConnectorApi { - MinioConnector minConnector = new MinioConnector(); + MinioConnector miniConnector = new MinioConnector(); HttpConnector ionosApi = new HttpConnector(); private MinioClient minioClient; @@ -38,11 +43,12 @@ public class S3ConnectorApiImpl implements S3ConnectorApi { public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, int maxFiles) { if(accessKey != null && secretKey != null && endpoint != null) - this.minioClient = minConnector.connect(endpoint, accessKey, secretKey); + this.minioClient = miniConnector.connect(endpoint, accessKey, secretKey); this.region = getRegion(endpoint); this.token = ""; this.maxFiles = maxFiles; } + public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, String token, int maxFiles) { this(endpoint, accessKey, secretKey, maxFiles); this.token = token; @@ -76,7 +82,7 @@ public boolean bucketExists(String bucketName) { } @Override - public void uploadObject(String bucketName, String objectName, InputStream stream) { + public void uploadObject(String bucketName, String objectName, ByteArrayInputStream stream) { if (!bucketExists(bucketName.toLowerCase())) { createBucket(bucketName.toLowerCase()); } @@ -94,17 +100,40 @@ public void uploadObject(String bucketName, String objectName, InputStream strea } @Override - public InputStream getObject(String bucketName, String objectName) { + public ByteArrayInputStream getObject(String bucketName, String objectName) { if (!bucketExists(bucketName.toLowerCase())) { throw new EdcException("Bucket not found - " + bucketName); } - try { - return minioClient.getObject(GetObjectArgs.builder() - .bucket(bucketName.toLowerCase()) - .region(region) - .object(objectName) - .build()); + var request = GetObjectArgs.builder() + .bucket(bucketName.toLowerCase()) + .region(region) + .object(objectName) + .build(); + + try (var response = minioClient.getObject(request)) { + return new ByteArrayInputStream(response.readAllBytes()); + } catch (Exception e) { + throw new EdcException("Getting file - " + e.getMessage()); + } + } + + @Override + public ByteArrayInputStream getObject(String bucketName, String objectName, long offset, long length) { + if (!bucketExists(bucketName.toLowerCase())) { + throw new EdcException("Bucket not found - " + bucketName); + } + + var request = GetObjectArgs.builder() + .bucket(bucketName.toLowerCase()) + .region(region) + .object(objectName) + .offset(offset) + .length(length) + .build(); + + try (var response = minioClient.getObject(request)) { + return new ByteArrayInputStream(response.readAllBytes()); } catch (Exception e) { throw new EdcException("Getting file - " + e.getMessage()); } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java index 1e49bc21..16bf41fd 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java @@ -16,6 +16,10 @@ public record S3Object(String objectName, long size) { + public boolean isDirectory() { + return objectName.endsWith("/"); + } + public boolean isRootObject(String blobName) { return (objectName.equals(blobName) || objectName.equals(blobName + "/")); } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java index 79439836..9413feed 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java @@ -17,11 +17,8 @@ import io.minio.MinioClient; public class MinioConnector { - public MinioClient connect(String endpoint, String accessKey, String secretKey) { - MinioClient minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); - - return minioClient; + return MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java index bbe76c21..be5f05a3 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java @@ -4,68 +4,61 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.FormBody; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; import org.eclipse.edc.spi.EdcException; - public class HttpConnector { - OkHttpClient client = new OkHttpClient(); - String basicUrl = "https://api.ionos.com/cloudapi/v6/um/users/"; - - public String retrieveUserID(String token) { - String[] jwtParts = token.split("\\."); - String jwtPayload = new String(java.util.Base64.getDecoder().decode(jwtParts[1])); - String uuid = jwtPayload.split("\"uuid\":\"")[1].split("\"")[0]; + OkHttpClient client = new OkHttpClient(); + String basicUrl = "https://api.ionos.com/cloudapi/v6/um/users/"; + + public String retrieveUserID(String token) { + String[] jwtParts = token.split("\\."); + String jwtPayload = new String(java.util.Base64.getDecoder().decode(jwtParts[1])); + String uuid = jwtPayload.split("\"uuid\":\"")[1].split("\"")[0]; + + return uuid; + } + + public TemporaryKey createTemporaryKey(String token) { + String url = basicUrl + retrieveUserID(token) + "/s3keys"; + + Request request = new Request.Builder().url(url) + //This adds the token to the header. + .addHeader("Authorization", "Bearer " + token) + .post(RequestBody.create(null, new byte[0])) + .build(); + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } - return uuid; - } - - - public TemporaryKey createTemporaryKey(String token) { - String url = basicUrl + retrieveUserID(token) + "/s3keys"; - - - Request request = new Request.Builder() - .url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) - .post(RequestBody.create(null, new byte[0])) - .build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()){ - throw new IOException("Unexpected code " + response); - } + ObjectMapper objectMapper = new ObjectMapper(); + S3Key resp = objectMapper.readValue(response.body().string(), S3Key.class); + TemporaryKey temp = new TemporaryKey(resp.getId().toString(), resp.getProperties().get("secretKey").toString()); + return temp; + } catch (IOException e) { + throw new EdcException("Error getting S3 temporary key", e); + } + } + public void deleteTemporaryAccount(String token, String keyID) { + String url = basicUrl + retrieveUserID(token) + "/s3keys/" + keyID; - ObjectMapper objectMapper = new ObjectMapper(); - S3Key resp = objectMapper.readValue(response.body().string(), S3Key.class); - TemporaryKey temp = new TemporaryKey(resp.getId().toString(),resp.getProperties().get("secretKey").toString()); - return temp; - } catch (IOException e) { - throw new EdcException("Error getting S3 temporary key", e); - } - } - - public void deleteTemporaryAccount(String token, String keyID) { - String url = basicUrl + retrieveUserID(token) + "/s3keys/" + keyID; + Request request = new Request.Builder().url(url) + //This adds the token to the header. + .addHeader("Authorization", "Bearer " + token) + .delete() + .build(); - Request request = new Request.Builder() - .url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) - .delete() - .build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()){ - throw new IOException("Unexpected code " + response); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response + " deleting S3 temporary key"); + } + } catch (IOException e) { + throw new EdcException("Error deleting S3 temporary key", e); + } + } } diff --git a/extensions/data-plane-ionos-s3/build.gradle.kts b/extensions/data-plane-ionos-s3/build.gradle.kts index 6cda551b..15e13a89 100644 --- a/extensions/data-plane-ionos-s3/build.gradle.kts +++ b/extensions/data-plane-ionos-s3/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { implementation("org.realityforge.org.jetbrains.annotations:org.jetbrains.annotations:1.7.0") testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.1") + testImplementation("org.junit.jupiter:junit-jupiter-engine:5.9.1") testImplementation("org.assertj:assertj-core:3.22.0") } @@ -38,6 +39,10 @@ java { withSourcesJar() } +tasks.test { + useJUnitPlatform() +} + publishing { publications { create("maven") { diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java index e1ba7263..9a244bbc 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java @@ -24,6 +24,8 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; + @Extension(value = DataPlaneIonosS3Extension.NAME) public class DataPlaneIonosS3Extension implements ServiceExtension { @@ -55,7 +57,7 @@ public void initialize(ServiceExtensionContext context) { var sourceFactory = new IonosDataSourceFactory(s3Api, monitor); pipelineService.registerFactory(sourceFactory); - var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager); + var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager, IONOS_MAX_FILES_DEFAULT); pipelineService.registerFactory(sinkFactory); context.getMonitor().info("File Transfer Extension initialized!"); } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java index 5fb6dbb2..668ef225 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java @@ -18,9 +18,10 @@ import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; -import org.eclipse.edc.spi.EdcException; import org.jetbrains.annotations.NotNull; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.List; import java.util.Objects; @@ -31,7 +32,7 @@ public class IonosDataSink extends ParallelSink { private S3ConnectorApi s3Api; private String bucketName; private String blobName; - + private IonosDataSink() {} @Override @@ -45,8 +46,25 @@ protected StreamResult transferParts(List parts) { blobName = part.name(); } - try (var input = part.openStream()) { - s3Api.uploadObject(bucketName, blobName, input); + var streamsOutput = new ByteArrayOutputStream(); + var stream = part.openStream(); + while (stream != null) { + try { + streamsOutput.write(stream.readAllBytes()); + stream.close(); + + } catch (Exception e) { + return uploadFailure(e, blobName); + } + + stream = part.openStream(); + } + + var byteArray = streamsOutput.toByteArray(); + try (var streamsInput = new ByteArrayInputStream(byteArray)) { + s3Api.uploadObject(bucketName, blobName, streamsInput); + streamsOutput.close(); + } catch (Exception e) { return uploadFailure(e, blobName); } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java index de77b7cb..c8bb7474 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java @@ -34,8 +34,6 @@ import java.util.concurrent.ExecutorService; -import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; - public class IonosDataSinkFactory implements DataSinkFactory { private static final String DEFAULT_STORAGE = "s3-eu-central-1.ionoscloud.com"; @@ -45,16 +43,18 @@ public class IonosDataSinkFactory implements DataSinkFactory { private final S3ConnectorApi s3Api; private final Vault vault; private final TypeManager typeManager; + private final int maxFiles; private final Validator validator = new IonosSinkDataAddressValidationRule(); - - public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, - Vault vault, TypeManager typeManager) { + + public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, Vault vault, + TypeManager typeManager, int maxFiles) { this.s3Api = s3Api; this.executorService = executorService; this.monitor = monitor; this.vault = vault; this.typeManager = typeManager; + this.maxFiles = maxFiles; } @Override @@ -87,20 +87,21 @@ public DataSink createSink(DataFlowRequest request) { token.getAccessKey(), token.getSecretKey(), "", - IONOS_MAX_FILES_DEFAULT); + maxFiles); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .blobName(destination.getStringProperty(IonosBucketSchema.BLOB_NAME)) .requestId(request.getId()) .executorService(executorService) - .monitor(monitor).s3Api(s3ApiTemp) + .monitor(monitor) + .s3Api(s3ApiTemp) .build(); } else { var s3ApiTemp = new S3ConnectorApiImpl(DEFAULT_STORAGE, token.getAccessKey(), token.getSecretKey(), "", - IONOS_MAX_FILES_DEFAULT); + maxFiles); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .blobName(destination.getStringProperty(IonosBucketSchema.BLOB_NAME)) @@ -116,7 +117,8 @@ public DataSink createSink(DataFlowRequest request) { .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .blobName(destination.getStringProperty(IonosBucketSchema.BLOB_NAME)) .requestId(request.getId()).executorService(executorService) - .monitor(monitor).s3Api(s3Api) + .monitor(monitor) + .s3Api(s3Api) .build(); } } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java index 85dc3f32..b9e2e106 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java @@ -14,11 +14,13 @@ package com.ionos.edc.dataplane.ionos.s3; +import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; import com.ionos.edc.extension.s3.api.S3ConnectorApi; import com.ionos.edc.extension.s3.api.S3Object; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; +import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.util.string.StringUtils; import org.eclipse.edc.spi.monitor.Monitor; @@ -73,15 +75,10 @@ public StreamResult> openPartStream() { ); } - List partStream = objects.stream() - .map(object -> new S3Part(this.s3Api, this.monitor, this.bucketName, object.objectName(), object.size())) + List parts = objects.stream() + .map(object -> new S3Part(s3Api, monitor, bucketName, object.objectName(), object.isDirectory(), object.size())) .collect(Collectors.toList()); - - return success(partStream.stream()); - } - - @Override - public void close() { + return success(parts.stream()); } boolean applyFilterIncludes(S3Object object) { @@ -97,20 +94,29 @@ boolean applyFilterExcludes(S3Object object) { return !filterExcludes.matcher(object.shortObjectName(blobName)).find(); } + @Override + public void close() { + } + private static class S3Part implements Part { private final S3ConnectorApi s3Api; private final Monitor monitor; private final String bucketName; private final String blobName; - private final long size; + private boolean isDirectory; + private final long fileSize; + + private boolean isOpened = true; + private long currentOffset = 0; - S3Part(S3ConnectorApi s3Api, Monitor monitor, String bucketName, String blobName, long size) { + S3Part(S3ConnectorApi s3Api, Monitor monitor, String bucketName, String blobName, boolean isDirectory, long fileSize) { super(); this.s3Api = s3Api; this.monitor = monitor; this.bucketName = bucketName; this.blobName = blobName; - this.size = size; + this.isDirectory = isDirectory; + this.fileSize = fileSize; } @Override @@ -119,12 +125,41 @@ public String name() { } @Override - public InputStream openStream() { - return s3Api.getObject(bucketName, blobName); + public long size() { + return fileSize; + } + + private long chunkSize() { + return FileTransferHelper.calculateChunkSize(fileSize); } @Override - public void close() { + public InputStream openStream() { + + if (isOpened && (isDirectory || (currentOffset >= fileSize))) + return null; + + InputStream stream; + if (isDirectory || (fileSize <= chunkSize())) { + stream = s3Api.getObject(bucketName, blobName); + } else { + stream = s3Api.getObject(bucketName, blobName, currentOffset, chunkSize()); + } + + if (!isDirectory) { + int responseSize; + try { + responseSize = stream.available(); + } catch (Exception e) { + throw new EdcException("Error reading response size", e); + } + + currentOffset += responseSize; + } + + if (!isOpened) + isOpened = true; + return stream; } } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelper.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelper.java new file mode 100644 index 00000000..3b1d1b17 --- /dev/null +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelper.java @@ -0,0 +1,37 @@ +package com.ionos.edc.dataplane.ionos.s3.util; + +import org.eclipse.edc.spi.EdcException; + +public class FileTransferHelper { + + private static final long MEGABYTE = 1048576; + + public static long calculateChunkSize(long fileSize) { + if (fileSize == 0) + return 0; + + if (fileSize <= MEGABYTE) { + return MEGABYTE; + } + + var fileSizeMB = fileSize / MEGABYTE; + + if ((fileSizeMB >= 1) && (fileSizeMB <= 20)) { + return MEGABYTE; + } else if ((fileSizeMB > 20) && (fileSizeMB <= 40)) { + return 2 * MEGABYTE; + } else if ((fileSizeMB > 40) && (fileSizeMB <= 60)) { + return 3 * MEGABYTE; + } else if ((fileSizeMB > 60) && (fileSizeMB <= 80)) { + return 4 * MEGABYTE; + } else if ((fileSizeMB > 80) && (fileSizeMB <= 100)) { + return 5 * MEGABYTE; + } else if ((fileSizeMB > 100) && (fileSizeMB <= 200)) { + return 10 * MEGABYTE; + } else if ((fileSizeMB > 200) && (fileSizeMB <= 300)) { + return 15 * MEGABYTE; + } else { + throw new EdcException("Unsupported file size: " + fileSize + " bytes"); + } + } +} \ No newline at end of file diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/GenerateBucket.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/GenerateBucket.java deleted file mode 100644 index 4da12af9..00000000 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/util/GenerateBucket.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.ionos.edc.dataplane.ionos.s3.util; - -import java.nio.charset.Charset; -import java.util.*; -public class GenerateBucket { - - public String genBucket() { - String CHARS = "abcdefghijklmnopqrstuvxyz1234567890"; - StringBuilder charGen = new StringBuilder(); - Random rnd = new Random(); - while (charGen.length() < 10) { // length of the random string. - int index = (int) (rnd.nextFloat() * CHARS.length()); - charGen.append(CHARS.charAt(index)); - } - String charGenString = charGen.toString(); - return charGenString; - - } - -} diff --git a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java b/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java new file mode 100644 index 00000000..d9baa788 --- /dev/null +++ b/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java @@ -0,0 +1,149 @@ +package org.eclipse.edc.connector.dataplane.ionos.s3.util; + +import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; +import org.eclipse.edc.spi.EdcException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class FileTransferHelperTest { + + @Test + public void calculateChunkSize_0Bytes() { + var chunkSize = FileTransferHelper.calculateChunkSize(0); + assertEquals(0, chunkSize); + } + + @Test + public void calculateChunkSize_1MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(1048576); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_2MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(2097152); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_3MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(3145728); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_4MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(4194304); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_5MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(5242880); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_6MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(6291456); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_7MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(7340032); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_8MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(8388608); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_9MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(9437184); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_10MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(10485760); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_20MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(20971520); + assertEquals(1048576, chunkSize); + } + + @Test + public void calculateChunkSize_30MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(31457280); + assertEquals(2097152, chunkSize); + } + + @Test + public void calculateChunkSize_40MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(41943040); + assertEquals(2097152, chunkSize); + } + + @Test + public void calculateChunkSize_50MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(52428800); + assertEquals(3145728, chunkSize); + } + + @Test + public void calculateChunkSize_60MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(62914560); + assertEquals(3145728, chunkSize); + } + + @Test + public void calculateChunkSize_70MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(73400320); + assertEquals(4194304, chunkSize); + } + + @Test + public void calculateChunkSize_80MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(83886080); + assertEquals(4194304, chunkSize); + } + + @Test + public void calculateChunkSize_90MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(94371840); + assertEquals(5242880, chunkSize); + } + + @Test + public void calculateChunkSize_100MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(104857600); + assertEquals(5242880, chunkSize); + } + + @Test + public void calculateChunkSize_200MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(209715200); + assertEquals(10485760, chunkSize); + } + + @Test + public void calculateChunkSize_300MB() { + var chunkSize = FileTransferHelper.calculateChunkSize(209715200); + assertEquals(10485760, chunkSize); + } + + @Test + public void calculateChunkSize_UnsupportedSize() { + assertThrows(EdcException.class, + ()-> FileTransferHelper.calculateChunkSize(10737418240L)); + } +} diff --git a/gradle.properties b/gradle.properties index da202f54..847b7f3b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ edcVersion=0.4.1 metaModelVersion=0.0.1-SNAPSHOT postgresVersion=42.6.0 rsApi=3.1.0 -minIOVersion=8.4.6 +minIOVersion=8.5.8 ionosDevelopersName=Paulo Lory, Paulo Cabrita gitHubPkgsName=GitHubPackages gitHubRpName=edc-ionos-s3