From d5852b5fb238027ed7bdc42d95821535d7ed95c7 Mon Sep 17 00:00:00 2001 From: ndr_brt Date: Mon, 21 Oct 2024 15:39:47 +0200 Subject: [PATCH] refactor(test): introduce Azurite and Minio extensions --- .../edc/tests/aws/MinioContainer.java | 42 ------ .../edc/tests/aws/MinioExtension.java | 69 ++++++++++ ...reBlobHelper.java => AzureBlobClient.java} | 4 +- .../edc/tests/azure/AzuriteContainer.java | 45 ------- .../edc/tests/azure/AzuriteExtension.java | 72 ++++++++++ .../transfer/test/AzureToAzureTest.java | 118 ++++++++++------- .../transfer/test/MultiCloudTest.java | 124 +++++++++--------- .../dataplane/transfer/test/S3ToS3Test.java | 88 +++++++------ .../transfer/test/TestConstants.java | 45 +------ .../transfer/AzureToAzureEndToEndTest.java | 79 +++++------ .../tests/transfer/S3ToS3EndToEndTest.java | 53 ++++---- 11 files changed, 395 insertions(+), 344 deletions(-) delete mode 100644 edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioContainer.java create mode 100644 edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioExtension.java rename edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/{AzureBlobHelper.java => AzureBlobClient.java} (97%) delete mode 100644 edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteContainer.java create mode 100644 edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteExtension.java diff --git a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioContainer.java b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioContainer.java deleted file mode 100644 index 134c06ff4..000000000 --- a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioContainer.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.eclipse.tractusx.edc.tests.aws; - -import org.testcontainers.containers.GenericContainer; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; - -import java.util.UUID; - -public class MinioContainer extends GenericContainer { - - private final String accessKeyId = "test-access-key"; - private final String secretAccessKey = UUID.randomUUID().toString(); - - public MinioContainer() { - super("bitnami/minio"); - addEnv("MINIO_ROOT_USER", accessKeyId); - addEnv("MINIO_ROOT_PASSWORD", secretAccessKey); - addExposedPort(9000); - } - - public AwsBasicCredentials getCredentials() { - return AwsBasicCredentials.create(accessKeyId, secretAccessKey); - } -} diff --git a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioExtension.java b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioExtension.java new file mode 100644 index 000000000..3d5970a48 --- /dev/null +++ b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/aws/MinioExtension.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024 Cofinity-X + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.eclipse.tractusx.edc.tests.aws; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.GenericContainer; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; + +import java.util.UUID; + +public class MinioExtension implements BeforeAllCallback, AfterAllCallback { + + private final MinioContainer minioContainer = new MinioContainer(); + + @Override + public void beforeAll(ExtensionContext context) { + minioContainer.start(); + } + + @Override + public void afterAll(ExtensionContext context) { + minioContainer.stop(); + } + + public int getPort() { + return minioContainer.getFirstMappedPort(); + } + + public AwsCredentials getCredentials() { + return minioContainer.getCredentials(); + } + + private static class MinioContainer extends GenericContainer { + + private final String accessKeyId = "test-access-key"; + private final String secretAccessKey = UUID.randomUUID().toString(); + + public MinioContainer() { + super("bitnami/minio"); + addEnv("MINIO_ROOT_USER", accessKeyId); + addEnv("MINIO_ROOT_PASSWORD", secretAccessKey); + addExposedPort(9000); + } + + public AwsBasicCredentials getCredentials() { + return AwsBasicCredentials.create(accessKeyId, secretAccessKey); + } + } +} diff --git a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobHelper.java b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobClient.java similarity index 97% rename from edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobHelper.java rename to edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobClient.java index d07dc5625..f43f198cd 100644 --- a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobHelper.java +++ b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzureBlobClient.java @@ -36,14 +36,14 @@ /** * Helper class that internally uses Azure SDK classes to create containers, upload blobs, generate SAS tokens, etc. */ -public class AzureBlobHelper { +public class AzureBlobClient { private final String accountName; private final String key; private final String host; private final int port; private BlobServiceClient blobServiceClient; - AzureBlobHelper(String accountName, String key, String host, int port) { + AzureBlobClient(String accountName, String key, String host, int port) { this.accountName = accountName; this.key = key; this.host = host; diff --git a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteContainer.java b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteContainer.java deleted file mode 100644 index 53fbd6c2e..000000000 --- a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteContainer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2024 Cofinity-X - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.eclipse.tractusx.edc.tests.azure; - -import org.testcontainers.containers.GenericContainer; - -import java.util.List; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.joining; - -public class AzuriteContainer extends GenericContainer { - - private static final String IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite"; - private final int containerPort = 10_000; - - public AzuriteContainer(int azuriteHostPort, Account... accounts) { - super(IMAGE_NAME); - addEnv("AZURITE_ACCOUNTS", stream(accounts).map(it -> "%s:%s".formatted(it.name, it.key)).collect(joining(";"))); - setPortBindings(List.of("%d:%d".formatted(azuriteHostPort, containerPort))); - } - - public AzureBlobHelper getHelper(Account account) { - return new AzureBlobHelper(account.name(), account.key(), getHost(), getMappedPort(containerPort)); - } - - public record Account(String name, String key) { } -} diff --git a/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteExtension.java b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteExtension.java new file mode 100644 index 000000000..94b07d511 --- /dev/null +++ b/edc-tests/e2e-fixtures/src/testFixtures/java/org/eclipse/tractusx/edc/tests/azure/AzuriteExtension.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024 Cofinity-X + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.eclipse.tractusx.edc.tests.azure; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.GenericContainer; + +import java.util.List; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.joining; + +public class AzuriteExtension implements BeforeAllCallback, AfterAllCallback { + + private final AzuriteContainer azuriteContainer; + + public AzuriteExtension(int azuriteHostPort, Account... accounts) { + azuriteContainer = new AzuriteContainer(azuriteHostPort, accounts); + } + + @Override + public void beforeAll(ExtensionContext context) { + azuriteContainer.start(); + } + + @Override + public void afterAll(ExtensionContext context) { + azuriteContainer.stop(); + } + + public AzureBlobClient getClientFor(Account account) { + return azuriteContainer.getHelper(account); + } + + public record Account(String name, String key) { } + + private static class AzuriteContainer extends GenericContainer { + + private static final String IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite"; + private final int containerPort = 10_000; + + public AzuriteContainer(int azuriteHostPort, Account... accounts) { + super(IMAGE_NAME); + addEnv("AZURITE_ACCOUNTS", stream(accounts).map(it -> "%s:%s".formatted(it.name(), it.key())).collect(joining(";"))); + setPortBindings(List.of("%d:%d".formatted(azuriteHostPort, containerPort))); + } + + public AzureBlobClient getHelper(Account account) { + return new AzureBlobClient(account.name(), account.key(), getHost(), getMappedPort(containerPort)); + } + + } +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java index 2c86ab99a..4bd0ec7ad 100644 --- a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java @@ -31,21 +31,20 @@ import org.eclipse.edc.spi.monitor.ConsoleMonitor; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.security.Vault; -import org.eclipse.tractusx.edc.tests.azure.AzureBlobHelper; -import org.eclipse.tractusx.edc.tests.azure.AzuriteContainer; +import org.eclipse.tractusx.edc.tests.azure.AzureBlobClient; +import org.eclipse.tractusx.edc.tests.azure.AzuriteExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.io.FileInputStream; import java.io.IOException; import java.time.Duration; import java.util.ArrayDeque; -import java.util.List; +import java.util.Arrays; import java.util.UUID; import java.util.stream.IntStream; @@ -54,16 +53,12 @@ import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; import static org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage.EDC_DATA_FLOW_START_MESSAGE_TYPE; import static org.eclipse.edc.util.io.Ports.getFreePort; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_CONTAINER_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_KEY_ALIAS; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_CONTAINER_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_KEY_ALIAS; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.CONSUMER_AZURITE_ACCOUNT; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PREFIX_FOR_MUTIPLE_FILES; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PROVIDER_AZURITE_ACCOUNT; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobDestinationAddress; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobSourceAddress; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.createSparseFile; import static org.mockito.ArgumentMatchers.contains; import static org.mockito.ArgumentMatchers.isA; @@ -96,36 +91,39 @@ public class AzureToAzureTest { * two different endpoint templates for provider and consumer. Endpoint templates are configured globally. * Also, the host-port must be fixed/deterministic, as the {@code PROVIDER_RUNTIME} needs to know it in advance */ - @Container - private final AzuriteContainer azuriteContainer = new AzuriteContainer(AZURITE_HOST_PORT, PROVIDER_AZURITE_ACCOUNT, CONSUMER_AZURITE_ACCOUNT); + @RegisterExtension + private static final AzuriteExtension AZURITE_CONTAINER = new AzuriteExtension(AZURITE_HOST_PORT, PROVIDER_AZURITE_ACCOUNT, CONSUMER_AZURITE_ACCOUNT); - private AzureBlobHelper providerBlobHelper; - private AzureBlobHelper consumerBlobHelper; + private AzureBlobClient providerBlobHelper; + private AzureBlobClient consumerBlobHelper; @BeforeEach void setup() { - providerBlobHelper = azuriteContainer.getHelper(PROVIDER_AZURITE_ACCOUNT); - consumerBlobHelper = azuriteContainer.getHelper(CONSUMER_AZURITE_ACCOUNT); + providerBlobHelper = AZURITE_CONTAINER.getClientFor(PROVIDER_AZURITE_ACCOUNT); + consumerBlobHelper = AZURITE_CONTAINER.getClientFor(CONSUMER_AZURITE_ACCOUNT); } @Test void transferMultipleFile_success(Vault vault) { - var sourceContainer = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + var sourceContainerName = UUID.randomUUID().toString(); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var filesNames = new ArrayDeque(); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); var fileNames = IntStream.rangeClosed(1, 2).mapToObj(i -> PREFIX_FOR_MUTIPLE_FILES + i + '_' + TESTFILE_NAME).toList(); fileNames.forEach(filename -> providerBlobHelper.uploadBlob(sourceContainer, fileData, filename)); - consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + consumerBlobHelper.createContainer(destinationContainerName); vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + """.formatted(consumerBlobHelper.generateAccountSas(destinationContainerName))); - var request = createFlowRequestBuilder("any") - .add("sourceDataAddress", blobSourceAddress(List.of(dspaceProperty(EDC_NAMESPACE + "blobPrefix", PREFIX_FOR_MUTIPLE_FILES)))) - .build(); + var request = createFlowRequestBuilder( + blobAddress(sourceContainerName, PROVIDER_AZURITE_ACCOUNT.name(), AZBLOB_PROVIDER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobPrefix", PREFIX_FOR_MUTIPLE_FILES)), + blobAddress(destinationContainerName, CONSUMER_AZURITE_ACCOUNT.name(), AZBLOB_CONSUMER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", "any")) + ).build(); given().when() .baseUri(START_DATAFLOW_URL) @@ -138,32 +136,36 @@ void transferMultipleFile_success(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .isNotEmpty() .containsAll(filesNames)); - } @Test void transferFile_success(Vault vault) { // upload file to provider's blob store - var sourceContainer = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + var sourceContainerName = UUID.randomUUID().toString(); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, TESTFILE_NAME); // create container in consumer's blob store - consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + consumerBlobHelper.createContainer(destinationContainerName); vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + """.formatted(consumerBlobHelper.generateAccountSas(destinationContainerName))); given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createFlowRequestBuilder(TESTFILE_NAME).build()) + .body(createFlowRequestBuilder( + blobAddress(sourceContainerName, PROVIDER_AZURITE_ACCOUNT.name(), AZBLOB_PROVIDER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)), + blobAddress(destinationContainerName, CONSUMER_AZURITE_ACCOUNT.name(), AZBLOB_CONSUMER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)) + ).build()) .post() .then() .log().ifValidationFails() @@ -171,7 +173,7 @@ void transferFile_success(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .isNotEmpty() .contains(TESTFILE_NAME)); @@ -193,7 +195,8 @@ void transferFile_success(Vault vault) { @ValueSource(longs = { 1024 * 1024 * 512, 1024L * 1024L * 1024L, /*1024L * 1024L * 1024L * 1024 takes extremely long!*/ }) void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException { // upload file to provider's blob store - var bcc = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + var sourceContainerName = UUID.randomUUID().toString(); + var bcc = providerBlobHelper.createContainer(sourceContainerName); // create random binary file of >1gb in size var blobName = "largeblob.bin"; @@ -201,17 +204,21 @@ void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException { providerBlobHelper.uploadBlob(bcc, new FileInputStream(f), blobName); // create container in consumer's blob store - consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + consumerBlobHelper.createContainer(destinationContainerName); vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + """.formatted(consumerBlobHelper.generateAccountSas(destinationContainerName))); given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createFlowRequestBuilder(blobName).build()) + .body(createFlowRequestBuilder( + blobAddress(sourceContainerName, PROVIDER_AZURITE_ACCOUNT.name(), AZBLOB_PROVIDER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", blobName)), + blobAddress(destinationContainerName, CONSUMER_AZURITE_ACCOUNT.name(), AZBLOB_CONSUMER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", blobName)) + ).build()) .post() .then() .log().ifValidationFails() @@ -220,7 +227,7 @@ void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException { await().pollInterval(Duration.ofSeconds(10)) .atMost(Duration.ofSeconds(120)) - .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .isNotEmpty() .contains(blobName)); @@ -230,25 +237,27 @@ void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException { void transferFolder_targetFolderNotExists_shouldCreate(Vault vault) { vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); - var sas = consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + var sas = consumerBlobHelper.generateAccountSas(destinationContainerName); vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} """.formatted(sas)); // create container in consumer's blob store - consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + consumerBlobHelper.createContainer(destinationContainerName); - var sourceContainer = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + var sourceContainerName = UUID.randomUUID().toString(); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/blob.bin"); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/blob2.bin"); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/blob3.bin"); - var request = createFlowRequestBuilder(TESTFILE_NAME) - .add("sourceDataAddress", blobSourceAddress(List.of(dspaceProperty(EDC_NAMESPACE + "blobPrefix", "folder/")))) - .add("destinationDataAddress", blobDestinationAddress(List.of(dspaceProperty(EDC_NAMESPACE + "folderName", "destfolder")))) - .build(); + var request = createFlowRequestBuilder( + blobAddress(sourceContainerName, PROVIDER_AZURITE_ACCOUNT.name(), AZBLOB_PROVIDER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobPrefix", "folder/")), + blobAddress(destinationContainerName, CONSUMER_AZURITE_ACCOUNT.name(), AZBLOB_CONSUMER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "folderName", "destfolder")) + ).build(); given().when() .baseUri(START_DATAFLOW_URL) @@ -261,27 +270,34 @@ void transferFolder_targetFolderNotExists_shouldCreate(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .isNotEmpty() .contains("destfolder/folder/blob.bin", "destfolder/folder/blob2.bin", "destfolder/folder/blob3.bin")); } @Test void transferFile_targetContainerNotExist_shouldFail(Vault vault) { - var sourceContainer = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + var sourceContainerName = UUID.randomUUID().toString(); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, TESTFILE_NAME); vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); + var destinationContainerName = UUID.randomUUID().toString(); vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + """.formatted(consumerBlobHelper.generateAccountSas(destinationContainerName))); given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createFlowRequestBuilder(TESTFILE_NAME).build()) + .body( + createFlowRequestBuilder( + blobAddress(sourceContainerName, PROVIDER_AZURITE_ACCOUNT.name(), AZBLOB_PROVIDER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)), + blobAddress(destinationContainerName, CONSUMER_AZURITE_ACCOUNT.name(), AZBLOB_CONSUMER_KEY_ALIAS, dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)) + ).build() + ) .post() .then() .log().ifValidationFails() @@ -293,18 +309,28 @@ void transferFile_targetContainerNotExist_shouldFail(Vault vault) { .severe(contains("Error creating blob %s on account %s".formatted(TESTFILE_NAME, CONSUMER_AZURITE_ACCOUNT.name())), isA(IOException.class))); } - private JsonObjectBuilder createFlowRequestBuilder(String blobName) { + private JsonObjectBuilder createFlowRequestBuilder(JsonObjectBuilder sourceDataAddress, JsonObjectBuilder destinationDataAddress) { return Json.createObjectBuilder() .add("@context", Json.createObjectBuilder().add("@vocab", EDC_NAMESPACE).add("dspace", "https://w3id.org/dspace/v0.8/")) .add("@type", EDC_DATA_FLOW_START_MESSAGE_TYPE) .add("@id", UUID.randomUUID().toString()) .add("processId", UUID.randomUUID().toString()) - .add("sourceDataAddress", blobSourceAddress(List.of(dspaceProperty(EDC_NAMESPACE + "blobName", blobName)))) - .add("destinationDataAddress", blobDestinationAddress(List.of(dspaceProperty(EDC_NAMESPACE + "blobName", blobName)))) + .add("sourceDataAddress", sourceDataAddress) + .add("destinationDataAddress", destinationDataAddress) .add("flowType", "PUSH") .add("transferTypeDestination", "AzureStorage-PUSH"); } + private JsonObjectBuilder blobAddress(String containerName, String accountName, String keyName, JsonObjectBuilder... additionalProperties) { + return Json.createObjectBuilder() + .add("dspace:endpointType", "AzureStorage") + .add("dspace:endpointProperties", Json.createArrayBuilder(Arrays.asList(additionalProperties)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) + .add(dspaceProperty(EDC_NAMESPACE + "account", accountName)) + .add(dspaceProperty(EDC_NAMESPACE + "keyName", keyName)) + ); + } + private JsonObjectBuilder dspaceProperty(String name, String value) { return Json.createObjectBuilder() .add("dspace:name", name) diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java index 6308c728d..582acf8d2 100644 --- a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java @@ -33,13 +33,12 @@ import org.eclipse.edc.junit.extensions.RuntimePerClassExtension; import org.eclipse.edc.junit.testfixtures.TestUtils; import org.eclipse.edc.spi.security.Vault; -import org.eclipse.tractusx.edc.tests.aws.MinioContainer; -import org.eclipse.tractusx.edc.tests.azure.AzureBlobHelper; -import org.eclipse.tractusx.edc.tests.azure.AzuriteContainer; +import org.eclipse.tractusx.edc.tests.aws.MinioExtension; +import org.eclipse.tractusx.edc.tests.azure.AzureBlobClient; +import org.eclipse.tractusx.edc.tests.azure.AzuriteExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -58,11 +57,9 @@ import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; import static org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage.EDC_DATA_FLOW_START_MESSAGE_TYPE; import static org.eclipse.edc.util.io.Ports.getFreePort; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_CONTAINER_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_KEY_ALIAS; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.CONSUMER_AZURITE_ACCOUNT; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PREFIX_FOR_MUTIPLE_FILES; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_CONSUMER_BUCKET_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.listObjects; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; @@ -72,15 +69,14 @@ public class MultiCloudTest { // S3 test constants public static final String REGION = Region.US_WEST_2.id(); - public static final String BUCKET_NAME = S3_CONSUMER_BUCKET_NAME; public static final String BLOB_KEY_ALIAS = AZBLOB_CONSUMER_KEY_ALIAS; private static final int AZURITE_HOST_PORT = getFreePort(); - private static final String BLOB_CONTAINER_NAME = AZBLOB_CONSUMER_CONTAINER_NAME; // General constants, containers etc. private static final int PROVIDER_CONTROL_PORT = getFreePort(); // port of the control api private static final String START_DATAFLOW_URL = "http://localhost:%s/control/v1/dataflows".formatted(PROVIDER_CONTROL_PORT); + @RegisterExtension protected static final RuntimeExtension DATAPLANE_RUNTIME = new RuntimePerClassExtension(new EmbeddedRuntime( "MultiCloud-Dataplane", @@ -88,39 +84,41 @@ public class MultiCloudTest { ":edc-tests:runtime:dataplane-cloud" )); - @Container - private final MinioContainer s3Container = new MinioContainer(); + @RegisterExtension + private static final MinioExtension MINIO_CONTAINER = new MinioExtension(); - @Container - private final AzuriteContainer azuriteContainer = new AzuriteContainer(AZURITE_HOST_PORT, CONSUMER_AZURITE_ACCOUNT); + @RegisterExtension + private static final AzuriteExtension AZURITE_CONTAINER = new AzuriteExtension(AZURITE_HOST_PORT, CONSUMER_AZURITE_ACCOUNT); - private AzureBlobHelper blobStoreHelper; + private AzureBlobClient blobStoreClient; private S3Client s3Client; private String s3EndpointOverride; @BeforeEach void setup() { - blobStoreHelper = azuriteContainer.getHelper(CONSUMER_AZURITE_ACCOUNT); - s3EndpointOverride = "http://localhost:%s/".formatted(s3Container.getFirstMappedPort()); + blobStoreClient = AZURITE_CONTAINER.getClientFor(CONSUMER_AZURITE_ACCOUNT); + s3EndpointOverride = "http://localhost:%s/".formatted(MINIO_CONTAINER.getPort()); var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(s3EndpointOverride)) - .credentialsProvider(s3Container::getCredentials) + .credentialsProvider(MINIO_CONTAINER::getCredentials) .build(); s3Client = new AwsClientProviderImpl(providerConfig).s3Client(S3ClientRequest.from(REGION, s3EndpointOverride)); } @Test void transferFile_azureToS3MultipleFiles(Vault vault) { - var sourceContainer = blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + var containerName = UUID.randomUUID().toString(); + var sourceContainer = blobStoreClient.createContainer(containerName); var filesNames = new ArrayDeque(); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); var fileNames = IntStream.rangeClosed(1, 2).mapToObj(i -> PREFIX_FOR_MUTIPLE_FILES + i + '_' + TESTFILE_NAME).toList(); - fileNames.forEach(filename -> blobStoreHelper.uploadBlob(sourceContainer, fileData, filename)); + fileNames.forEach(filename -> blobStoreClient.uploadBlob(sourceContainer, fileData, filename)); vault.storeSecret(BLOB_KEY_ALIAS, CONSUMER_AZURITE_ACCOUNT.key()); - var destinationBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + var bucketName = UUID.randomUUID().toString(); + var destinationBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); assertThat(destinationBucket.sdkHttpResponse().isSuccessful()).isTrue(); var request = Json.createObjectBuilder() @@ -131,7 +129,7 @@ void transferFile_azureToS3MultipleFiles(Vault vault) { .add("sourceDataAddress", Json.createObjectBuilder() .add("dspace:endpointType", "AzureStorage") .add("dspace:endpointProperties", Json.createArrayBuilder() - .add(dspaceProperty(EDC_NAMESPACE + "container", BLOB_CONTAINER_NAME)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) .add(dspaceProperty(EDC_NAMESPACE + "keyName", BLOB_KEY_ALIAS)) .add(dspaceProperty(EDC_NAMESPACE + "blobPrefix", PREFIX_FOR_MUTIPLE_FILES)) @@ -141,9 +139,9 @@ void transferFile_azureToS3MultipleFiles(Vault vault) { .add("dspace:endpointType", S3BucketSchema.TYPE) .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, s3Container.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, s3Container.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, bucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, MINIO_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, MINIO_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride)) ) ) @@ -163,7 +161,7 @@ void transferFile_azureToS3MultipleFiles(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(listObjects(s3Client, BUCKET_NAME)) + .untilAsserted(() -> assertThat(listObjects(s3Client, bucketName)) .isNotEmpty() .containsAll(filesNames)); } @@ -171,11 +169,13 @@ void transferFile_azureToS3MultipleFiles(Vault vault) { @Test void transferFile_azureToS3(Vault vault) { var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); - var sourceContainer = blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); - blobStoreHelper.uploadBlob(sourceContainer, fileData, TESTFILE_NAME); + var containerName = UUID.randomUUID().toString(); + var sourceContainer = blobStoreClient.createContainer(containerName); + blobStoreClient.uploadBlob(sourceContainer, fileData, TESTFILE_NAME); vault.storeSecret(BLOB_KEY_ALIAS, CONSUMER_AZURITE_ACCOUNT.key()); - var r = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + var bucketName = UUID.randomUUID().toString(); + var r = s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); assertThat(r.sdkHttpResponse().isSuccessful()).isTrue(); var request = Json.createObjectBuilder() @@ -186,7 +186,7 @@ void transferFile_azureToS3(Vault vault) { .add("sourceDataAddress", Json.createObjectBuilder() .add("dspace:endpointType", "AzureStorage") .add("dspace:endpointProperties", Json.createArrayBuilder() - .add(dspaceProperty(EDC_NAMESPACE + "container", BLOB_CONTAINER_NAME)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) .add(dspaceProperty(EDC_NAMESPACE + "keyName", BLOB_KEY_ALIAS)) .add(dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)) @@ -196,9 +196,9 @@ void transferFile_azureToS3(Vault vault) { .add("dspace:endpointType", S3BucketSchema.TYPE) .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, s3Container.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, s3Container.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, bucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, MINIO_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, MINIO_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride)) ) ) @@ -217,7 +217,7 @@ void transferFile_azureToS3(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(listObjects(s3Client, BUCKET_NAME)) + .untilAsserted(() -> assertThat(listObjects(s3Client, bucketName)) .isNotEmpty() .contains(TESTFILE_NAME)); } @@ -225,7 +225,8 @@ void transferFile_azureToS3(Vault vault) { @Test void transferFile_s3ToAzureMultipleFiles(Vault vault) { - var sourceBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + var bucketName = UUID.randomUUID().toString(); + var sourceBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); assertThat(sourceBucket.sdkHttpResponse().isSuccessful()).isTrue(); var putResponse = new AtomicBoolean(true); @@ -233,17 +234,18 @@ void transferFile_s3ToAzureMultipleFiles(Vault vault) { var fileNames = IntStream.rangeClosed(1, 2).mapToObj(i -> PREFIX_FOR_MUTIPLE_FILES + i + '_' + TESTFILE_NAME).toList(); fileNames.forEach(filename -> putResponse.set(s3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET_NAME) + .bucket(bucketName) .key(filename) .build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()) .sdkHttpResponse() .isSuccessful() && putResponse.get())); assertThat(putResponse.get()).isTrue(); - blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + var containerName = UUID.randomUUID().toString(); + blobStoreClient.createContainer(containerName); vault.storeSecret(BLOB_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(blobStoreHelper.generateAccountSas(BLOB_CONTAINER_NAME))); + """.formatted(blobStoreClient.generateAccountSas(containerName))); var request = Json.createObjectBuilder() .add("@context", Json.createObjectBuilder().add("@vocab", EDC_NAMESPACE).add("dspace", "https://w3id.org/dspace/v0.8/")) @@ -255,16 +257,16 @@ void transferFile_s3ToAzureMultipleFiles(Vault vault) { .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_PREFIX, PREFIX_FOR_MUTIPLE_FILES)) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, s3Container.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, s3Container.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, bucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, MINIO_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, MINIO_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride)) ) ) .add("destinationDataAddress", Json.createObjectBuilder() .add("dspace:endpointType", "AzureStorage") .add("dspace:endpointProperties", Json.createArrayBuilder() - .add(dspaceProperty(EDC_NAMESPACE + "container", AZBLOB_CONSUMER_CONTAINER_NAME)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) .add(dspaceProperty(EDC_NAMESPACE + "keyName", AZBLOB_CONSUMER_KEY_ALIAS)) ) @@ -283,14 +285,15 @@ void transferFile_s3ToAzureMultipleFiles(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(blobStoreHelper.listBlobs(BLOB_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(blobStoreClient.listBlobs(containerName)) .isNotEmpty() .containsAll(filesNames)); } @Test void transferFile_s3ToAzureMultipleFiles_whenConsumerDefinesBloblName_success(Vault vault) { - var sourceBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + var bucketName = UUID.randomUUID().toString(); + var sourceBucket = s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); assertThat(sourceBucket.sdkHttpResponse().isSuccessful()).isTrue(); var putResponse = new AtomicBoolean(true); @@ -298,7 +301,7 @@ void transferFile_s3ToAzureMultipleFiles_whenConsumerDefinesBloblName_success(Va var fileNames = IntStream.rangeClosed(1, 2).mapToObj(i -> PREFIX_FOR_MUTIPLE_FILES + i + '_' + TESTFILE_NAME).toList(); fileNames.forEach(filename -> putResponse.set(s3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET_NAME) + .bucket(bucketName) .key(filename) .build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()) .sdkHttpResponse() @@ -306,10 +309,11 @@ void transferFile_s3ToAzureMultipleFiles_whenConsumerDefinesBloblName_success(Va assertThat(putResponse.get()).isTrue(); - blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + var containerName = UUID.randomUUID().toString(); + blobStoreClient.createContainer(containerName); vault.storeSecret(BLOB_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(blobStoreHelper.generateAccountSas(BLOB_CONTAINER_NAME))); + """.formatted(blobStoreClient.generateAccountSas(containerName))); var request = Json.createObjectBuilder() .add("@context", Json.createObjectBuilder().add("@vocab", EDC_NAMESPACE).add("dspace", "https://w3id.org/dspace/v0.8/")) @@ -321,16 +325,16 @@ void transferFile_s3ToAzureMultipleFiles_whenConsumerDefinesBloblName_success(Va .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_PREFIX, PREFIX_FOR_MUTIPLE_FILES)) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, s3Container.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, s3Container.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, bucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, MINIO_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, MINIO_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride)) ) ) .add("destinationDataAddress", Json.createObjectBuilder() .add("dspace:endpointType", "AzureStorage") .add("dspace:endpointProperties", Json.createArrayBuilder() - .add(dspaceProperty(EDC_NAMESPACE + "container", AZBLOB_CONSUMER_CONTAINER_NAME)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) .add(dspaceProperty(EDC_NAMESPACE + "keyName", AZBLOB_CONSUMER_KEY_ALIAS)) .add(dspaceProperty(EDC_NAMESPACE + "blobName", "NOME_TEST")) @@ -351,22 +355,24 @@ void transferFile_s3ToAzureMultipleFiles_whenConsumerDefinesBloblName_success(Va await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(blobStoreHelper.listBlobs(BLOB_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(blobStoreClient.listBlobs(containerName)) .isNotEmpty() .containsAll(filesNames)); } @Test void transferFile_s3ToAzure(Vault vault) { - var b1 = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + var bucketName = UUID.randomUUID().toString(); + var b1 = s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); - var putResponse = s3Client.putObject(PutObjectRequest.builder().bucket(BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + var putResponse = s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); - blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + var containerName = UUID.randomUUID().toString(); + blobStoreClient.createContainer(containerName); vault.storeSecret(BLOB_KEY_ALIAS, """ {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} - """.formatted(blobStoreHelper.generateAccountSas(BLOB_CONTAINER_NAME))); + """.formatted(blobStoreClient.generateAccountSas(containerName))); var request = Json.createObjectBuilder() .add("@context", Json.createObjectBuilder().add("@vocab", EDC_NAMESPACE).add("dspace", "https://w3id.org/dspace/v0.8/")) @@ -378,16 +384,16 @@ void transferFile_s3ToAzure(Vault vault) { .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_NAME, TESTFILE_NAME)) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, s3Container.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, s3Container.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, bucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, MINIO_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, MINIO_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride)) ) ) .add("destinationDataAddress", Json.createObjectBuilder() .add("dspace:endpointType", "AzureStorage") .add("dspace:endpointProperties", Json.createArrayBuilder() - .add(dspaceProperty(EDC_NAMESPACE + "container", AZBLOB_CONSUMER_CONTAINER_NAME)) + .add(dspaceProperty(EDC_NAMESPACE + "container", containerName)) .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) .add(dspaceProperty(EDC_NAMESPACE + "keyName", AZBLOB_CONSUMER_KEY_ALIAS)) .add(dspaceProperty(EDC_NAMESPACE + "blobName", TESTFILE_NAME)) @@ -407,7 +413,7 @@ void transferFile_s3ToAzure(Vault vault) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(blobStoreHelper.listBlobs(BLOB_CONTAINER_NAME)) + .untilAsserted(() -> assertThat(blobStoreClient.listBlobs(containerName)) .isNotEmpty() .contains(TESTFILE_NAME)); } diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java index f55161b46..6261fc486 100644 --- a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java @@ -34,15 +34,15 @@ import org.eclipse.edc.junit.testfixtures.TestUtils; import org.eclipse.edc.spi.monitor.ConsoleMonitor; import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.tractusx.edc.tests.aws.MinioContainer; +import org.eclipse.tractusx.edc.tests.aws.MinioExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; @@ -67,9 +67,6 @@ import static org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage.EDC_DATA_FLOW_START_MESSAGE_TYPE; import static org.eclipse.edc.util.io.Ports.getFreePort; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PREFIX_FOR_MUTIPLE_FILES; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_CONSUMER_BUCKET_NAME; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_PROVIDER_BUCKET_NAME; -import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_REGION; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.createSparseFile; import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.listObjects; @@ -87,6 +84,7 @@ @EndToEndTest public class S3ToS3Test { + private static final String S3_REGION = Region.US_WEST_2.id(); private static final int PROVIDER_CONTROL_PORT = getFreePort(); // port of the control api private static final String START_DATAFLOW_URL = "http://localhost:%s/control/v1/dataflows".formatted(PROVIDER_CONTROL_PORT); @@ -96,10 +94,12 @@ public class S3ToS3Test { RuntimeConfig.S3.s3dataplaneConfig("/control", PROVIDER_CONTROL_PORT), ":edc-tests:runtime:dataplane-cloud" )).registerServiceMock(Monitor.class, spy(new ConsoleMonitor("AwsS3-Dataplane", ConsoleMonitor.Level.DEBUG, true))); - @Container - private final MinioContainer providerContainer = new MinioContainer(); - @Container - private final MinioContainer consumerContainer = new MinioContainer(); + + @RegisterExtension + private static final MinioExtension PROVIDER_CONTAINER = new MinioExtension(); + + @RegisterExtension + private static final MinioExtension CONSUMER_CONTAINER = new MinioExtension(); private S3Client providerClient; private S3Client consumerClient; @@ -108,31 +108,32 @@ public class S3ToS3Test { @BeforeEach void setup() { - providerEndpointOverride = "http://localhost:%s/".formatted(providerContainer.getFirstMappedPort()); + providerEndpointOverride = "http://localhost:%s/".formatted(PROVIDER_CONTAINER.getPort()); var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(providerEndpointOverride)) - .credentialsProvider(providerContainer::getCredentials) + .credentialsProvider(PROVIDER_CONTAINER::getCredentials) .build(); providerClient = new AwsClientProviderImpl(providerConfig).s3Client(S3ClientRequest.from(S3_REGION, providerEndpointOverride)); - consumerEndpointOverride = "http://localhost:%s".formatted(consumerContainer.getFirstMappedPort()); + consumerEndpointOverride = "http://localhost:%s".formatted(CONSUMER_CONTAINER.getPort()); var consumerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(consumerEndpointOverride)) - .credentialsProvider(consumerContainer::getCredentials) + .credentialsProvider(CONSUMER_CONTAINER::getCredentials) .build(); consumerClient = new AwsClientProviderImpl(consumerConfig).s3Client(S3ClientRequest.from(S3_REGION, consumerEndpointOverride)); } @Test void transferMultipleFiles() { - var sourceBucket = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + var sourceBucketName = UUID.randomUUID().toString(); + var sourceBucket = providerClient.createBucket(CreateBucketRequest.builder().bucket(sourceBucketName).build()); assertThat(sourceBucket.sdkHttpResponse().isSuccessful()).isTrue(); var putResponse = new AtomicBoolean(true); var filesNames = new ArrayDeque(); var fileNames = IntStream.rangeClosed(1, 2).mapToObj(i -> PREFIX_FOR_MUTIPLE_FILES + i + '_' + TESTFILE_NAME).toList(); fileNames.forEach(filename -> putResponse.set(providerClient.putObject(PutObjectRequest.builder() - .bucket(S3_PROVIDER_BUCKET_NAME) + .bucket(sourceBucketName) .key(filename) .build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()) .sdkHttpResponse() @@ -140,14 +141,15 @@ void transferMultipleFiles() { assertThat(putResponse.get()).isTrue(); - var destinationBucket = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + var destinationBucketName = UUID.randomUUID().toString(); + var destinationBucket = consumerClient.createBucket(CreateBucketRequest.builder().bucket(destinationBucketName).build()); assertThat(destinationBucket.sdkHttpResponse().isSuccessful()).isTrue(); var additionalSourceAddressProperties = List.of(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_PREFIX, PREFIX_FOR_MUTIPLE_FILES)); given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createDataFlowStartMessage(additionalSourceAddressProperties)) + .body(createDataFlowStartMessage(sourceBucketName, destinationBucketName, additionalSourceAddressProperties)) .post() .then() .statusCode(200); @@ -155,24 +157,23 @@ void transferMultipleFiles() { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) .untilAsserted(() -> - assertThat(listObjects(consumerClient, S3_CONSUMER_BUCKET_NAME)) + assertThat(listObjects(consumerClient, destinationBucketName)) .isNotEmpty() .containsAll(filesNames)); } - @Test void transferFile_success() { - - // create bucket in provider - var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + var sourceBucketName = UUID.randomUUID().toString(); + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(sourceBucketName).build()); assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); // upload test file in provider - var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(sourceBucketName).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); // create bucket in consumer - var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + var destinationBucketName = UUID.randomUUID().toString(); + var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(destinationBucketName).build()); assertThat(b2.sdkHttpResponse().isSuccessful()).isTrue(); var additionalSourceAddressProperties = List.of(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_NAME, TESTFILE_NAME)); @@ -180,7 +181,7 @@ void transferFile_success() { given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createDataFlowStartMessage(additionalSourceAddressProperties)) + .body(createDataFlowStartMessage(sourceBucketName, destinationBucketName, additionalSourceAddressProperties)) .post() .then() .statusCode(200); @@ -188,7 +189,7 @@ void transferFile_success() { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(listObjects(consumerClient, S3_CONSUMER_BUCKET_NAME)) + .untilAsserted(() -> assertThat(listObjects(consumerClient, destinationBucketName)) .isNotEmpty() .contains(TESTFILE_NAME)); } @@ -196,10 +197,11 @@ void transferFile_success() { @Test void transferFile_targetContainerNotExist_shouldFail() { // create bucket in provider - var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + var sourceBucketName = UUID.randomUUID().toString(); + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(sourceBucketName).build()); assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); // upload test file in provider - var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(sourceBucketName).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); // do not create bucket in consumer -> will fail! @@ -209,7 +211,7 @@ void transferFile_targetContainerNotExist_shouldFail() { given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createDataFlowStartMessage(additionalSourceAddressProperties)) + .body(createDataFlowStartMessage(sourceBucketName, "not-existent-bucket", additionalSourceAddressProperties)) .post() .then() .statusCode(200); @@ -231,12 +233,14 @@ void transferfile_largeFile(long sizeBytes) { var file = createSparseFile(sizeBytes); // create bucket in provider - var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + var sourceBucketName = UUID.randomUUID().toString(); + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(sourceBucketName).build()); assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); // upload test file in provider - uploadLargeFile(file, S3_PROVIDER_BUCKET_NAME, TESTFILE_NAME).thenAccept(completedUpload -> { + uploadLargeFile(file, sourceBucketName, TESTFILE_NAME).thenAccept(completedUpload -> { // create bucket in consumer - var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + var destinationBucketName = UUID.randomUUID().toString(); + var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(destinationBucketName).build()); assertThat(b2.sdkHttpResponse().isSuccessful()).isTrue(); var additionalSourceAddressProperties = List.of(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_NAME, TESTFILE_NAME)); @@ -244,7 +248,7 @@ void transferfile_largeFile(long sizeBytes) { given().when() .baseUri(START_DATAFLOW_URL) .contentType(ContentType.JSON) - .body(createDataFlowStartMessage(additionalSourceAddressProperties)) + .body(createDataFlowStartMessage(sourceBucketName, destinationBucketName, additionalSourceAddressProperties)) .post() .then() .statusCode(200); @@ -252,7 +256,7 @@ void transferfile_largeFile(long sizeBytes) { await().pollInterval(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(60)) - .untilAsserted(() -> assertThat(listObjects(consumerClient, S3_CONSUMER_BUCKET_NAME)) + .untilAsserted(() -> assertThat(listObjects(consumerClient, destinationBucketName)) .isNotEmpty() .contains(TESTFILE_NAME)); }); @@ -263,7 +267,7 @@ private CompletableFuture uploadLargeFile(File file, String buc var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(providerEndpointOverride)) - .credentialsProvider(providerContainer::getCredentials) + .credentialsProvider(PROVIDER_CONTAINER::getCredentials) .build(); var asyncClient = new AwsClientProviderImpl(providerConfig).s3AsyncClient(S3_REGION); var tm = S3TransferManager.builder() @@ -280,7 +284,7 @@ private CompletableFuture uploadLargeFile(File file, String buc return upload.completionFuture(); } - private JsonObject createDataFlowStartMessage(List additionalSourceAddressProperties) { + private JsonObject createDataFlowStartMessage(String sourceBucketName, String destinationBucketName, List additionalSourceAddressProperties) { return Json.createObjectBuilder() .add("@context", Json.createObjectBuilder().add("@vocab", EDC_NAMESPACE).add("dspace", "https://w3id.org/dspace/v0.8/")) .add("@type", EDC_DATA_FLOW_START_MESSAGE_TYPE) @@ -290,9 +294,9 @@ private JsonObject createDataFlowStartMessage(List additional .add("dspace:endpointType", S3BucketSchema.TYPE) .add("dspace:endpointProperties", Json.createArrayBuilder(additionalSourceAddressProperties) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, S3_REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, S3_PROVIDER_BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, providerContainer.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, providerContainer.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, sourceBucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, PROVIDER_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, PROVIDER_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, providerEndpointOverride)) ) ) @@ -301,9 +305,9 @@ private JsonObject createDataFlowStartMessage(List additional .add("dspace:endpointProperties", Json.createArrayBuilder() .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.OBJECT_NAME, TESTFILE_NAME)) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.REGION, S3_REGION)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, S3_CONSUMER_BUCKET_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, consumerContainer.getCredentials().accessKeyId())) - .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, consumerContainer.getCredentials().secretAccessKey())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.BUCKET_NAME, destinationBucketName)) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ACCESS_KEY_ID, CONSUMER_CONTAINER.getCredentials().accessKeyId())) + .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.SECRET_ACCESS_KEY, CONSUMER_CONTAINER.getCredentials().secretAccessKey())) .add(dspaceProperty(EDC_NAMESPACE + S3BucketSchema.ENDPOINT_OVERRIDE, consumerEndpointOverride)) ) ) diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java index ea148e9e8..5d1a1e60a 100644 --- a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java @@ -19,57 +19,18 @@ package org.eclipse.tractusx.edc.dataplane.transfer.test; -import jakarta.json.Json; -import jakarta.json.JsonObjectBuilder; -import org.eclipse.tractusx.edc.tests.azure.AzuriteContainer; -import software.amazon.awssdk.regions.Region; - -import java.util.List; - -import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; +import org.eclipse.tractusx.edc.tests.azure.AzuriteExtension; public class TestConstants { // AZURE BLOB CONSTANTS - public static final AzuriteContainer.Account PROVIDER_AZURITE_ACCOUNT = new AzuriteContainer.Account("provider", "providerkey"); - public static final AzuriteContainer.Account CONSUMER_AZURITE_ACCOUNT = new AzuriteContainer.Account("consumer", "consumerkey"); + public static final AzuriteExtension.Account PROVIDER_AZURITE_ACCOUNT = new AzuriteExtension.Account("provider", "providerkey"); + public static final AzuriteExtension.Account CONSUMER_AZURITE_ACCOUNT = new AzuriteExtension.Account("consumer", "consumerkey"); // alias under which the provider key is stored in the vault. must end with -key1 public static final String AZBLOB_PROVIDER_KEY_ALIAS = "provider-key1"; // alias under which the consumer key is stored in the vault. must end with -key1 public static final String AZBLOB_CONSUMER_KEY_ALIAS = "consumer-key1"; - public static final String AZBLOB_PROVIDER_CONTAINER_NAME = "src-container"; - public static final String AZBLOB_CONSUMER_CONTAINER_NAME = "dest-container"; - // AMAZON S3 CONSTANTS - public static final String S3_REGION = Region.US_WEST_2.id(); - public static final String S3_PROVIDER_BUCKET_NAME = "provider-bucket"; - public static final String S3_CONSUMER_BUCKET_NAME = "consumer-bucket"; // GENERIC CONSTANTS public static final String TESTFILE_NAME = "testfile.json"; public static final String PREFIX_FOR_MUTIPLE_FILES = "m/"; - public static JsonObjectBuilder blobSourceAddress(List additionalProperties) { - return Json.createObjectBuilder() - .add("dspace:endpointType", "AzureStorage") - .add("dspace:endpointProperties", Json.createArrayBuilder(additionalProperties) - .add(dspaceProperty(EDC_NAMESPACE + "container", AZBLOB_PROVIDER_CONTAINER_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + "account", PROVIDER_AZURITE_ACCOUNT.name())) - .add(dspaceProperty(EDC_NAMESPACE + "keyName", AZBLOB_PROVIDER_KEY_ALIAS)) - ); - } - - public static JsonObjectBuilder blobDestinationAddress(List additionalProperties) { - return Json.createObjectBuilder() - .add("dspace:endpointType", "AzureStorage") - .add("dspace:endpointProperties", Json.createArrayBuilder(additionalProperties) - .add(dspaceProperty(EDC_NAMESPACE + "container", AZBLOB_CONSUMER_CONTAINER_NAME)) - .add(dspaceProperty(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name())) - .add(dspaceProperty(EDC_NAMESPACE + "keyName", AZBLOB_CONSUMER_KEY_ALIAS)) - ); - } - - private static JsonObjectBuilder dspaceProperty(String name, String value) { - return Json.createObjectBuilder() - .add("dspace:name", name) - .add("dspace:value", value); - } - } diff --git a/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/AzureToAzureEndToEndTest.java b/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/AzureToAzureEndToEndTest.java index b18776fce..bbcd0847b 100644 --- a/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/AzureToAzureEndToEndTest.java +++ b/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/AzureToAzureEndToEndTest.java @@ -25,18 +25,18 @@ import org.eclipse.edc.junit.extensions.RuntimeExtension; import org.eclipse.edc.junit.testfixtures.TestUtils; import org.eclipse.edc.spi.security.Vault; -import org.eclipse.tractusx.edc.tests.azure.AzureBlobHelper; -import org.eclipse.tractusx.edc.tests.azure.AzuriteContainer; +import org.eclipse.tractusx.edc.tests.azure.AzureBlobClient; +import org.eclipse.tractusx.edc.tests.azure.AzuriteExtension; import org.eclipse.tractusx.edc.tests.participant.TractusxParticipantBase; import org.eclipse.tractusx.edc.tests.participant.TransferParticipant; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.util.HashMap; import java.util.Map; +import java.util.UUID; import static jakarta.json.Json.createObjectBuilder; import static org.assertj.core.api.Assertions.assertThat; @@ -61,9 +61,6 @@ @EndToEndTest public class AzureToAzureEndToEndTest { - public static final String AZURITE_DOCKER_IMAGE = "mcr.microsoft.com/azure-storage/azurite"; - public static final String CONSUMER_CONTAINER_NAME = "consumer-container"; - public static final String PROVIDER_CONTAINER_NAME = "provider-container"; public static final String PROVIDER_KEY_ALIAS = "provider-key-alias"; protected static final TransferParticipant CONSUMER = TransferParticipant.Builder.newInstance() .name(CONSUMER_NAME) @@ -78,14 +75,17 @@ public class AzureToAzureEndToEndTest { protected static final RuntimeExtension PROVIDER_RUNTIME = memoryRuntime(PROVIDER.getName(), PROVIDER.getBpn(), with(PROVIDER.getConfiguration(), AZURITE_HOST_PORT)); @RegisterExtension protected static final RuntimeExtension CONSUMER_RUNTIME = memoryRuntime(CONSUMER.getName(), CONSUMER.getBpn(), with(CONSUMER.getConfiguration(), AZURITE_HOST_PORT)); + + private static final AzuriteExtension.Account PROVIDER_AZURITE_ACCOUNT = new AzuriteExtension.Account("provider", Base64.encodeBase64String("provider-key".getBytes())); + private static final AzuriteExtension.Account CONSUMER_AZURITE_ACCOUNT = new AzuriteExtension.Account("consumer", Base64.encodeBase64String("consumer-key".getBytes())); + + @RegisterExtension + private static final AzuriteExtension AZURITE_CONTAINER = new AzuriteExtension(AZURITE_HOST_PORT, PROVIDER_AZURITE_ACCOUNT, CONSUMER_AZURITE_ACCOUNT); + private static final String TESTFILE_NAME = "hello.txt"; - private final AzuriteContainer.Account providerAzuriteAccount = new AzuriteContainer.Account("provider", Base64.encodeBase64String("provider-key".getBytes())); - private final AzuriteContainer.Account consumerAzuriteAccount = new AzuriteContainer.Account("consumer", Base64.encodeBase64String("consumer-key".getBytes())); - @Container - private final AzuriteContainer azuriteContainer = new AzuriteContainer(AZURITE_HOST_PORT, providerAzuriteAccount, consumerAzuriteAccount); - private AzureBlobHelper providerBlobHelper; - private AzureBlobHelper consumerBlobHelper; + private AzureBlobClient providerBlobHelper; + private AzureBlobClient consumerBlobHelper; private static Map with(Map configuration, int port) { configuration.putAll(new HashMap<>() { @@ -107,34 +107,36 @@ public TractusxParticipantBase consumer() { @BeforeEach void setup() { PROVIDER_RUNTIME.getService(Vault.class) - .storeSecret(PROVIDER_KEY_ALIAS, providerAzuriteAccount.key()); + .storeSecret(PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key()); CONSUMER_RUNTIME.getService(Vault.class) - .storeSecret("%s-key1".formatted(consumerAzuriteAccount.name()), consumerAzuriteAccount.key()); + .storeSecret("%s-key1".formatted(CONSUMER_AZURITE_ACCOUNT.name()), CONSUMER_AZURITE_ACCOUNT.key()); - providerBlobHelper = azuriteContainer.getHelper(providerAzuriteAccount); - consumerBlobHelper = azuriteContainer.getHelper(consumerAzuriteAccount); + providerBlobHelper = AZURITE_CONTAINER.getClientFor(PROVIDER_AZURITE_ACCOUNT); + consumerBlobHelper = AZURITE_CONTAINER.getClientFor(CONSUMER_AZURITE_ACCOUNT); } @Test void azureBlobPush_withDestFolder() { var assetId = "felix-blob-test-asset"; + var sourceContainerName = UUID.randomUUID().toString(); Map dataAddress = Map.of( "name", "transfer-test", "@type", "DataAddress", "type", "AzureStorage", - "container", PROVIDER_CONTAINER_NAME, - "account", providerAzuriteAccount.name(), + "container", sourceContainerName, + "account", PROVIDER_AZURITE_ACCOUNT.name(), "blobPrefix", "folder/", "keyName", PROVIDER_KEY_ALIAS ); // upload file to provider's blob store - var sourceContainer = providerBlobHelper.createContainer(PROVIDER_CONTAINER_NAME); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/" + TESTFILE_NAME); - consumerBlobHelper.createContainer(CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + consumerBlobHelper.createContainer(destinationContainerName); // create objects in EDC provider().createAsset(assetId, Map.of(), dataAddress); @@ -147,8 +149,8 @@ void azureBlobPush_withDestFolder() { .add(EDC_NAMESPACE + "type", "AzureStorage") .add(EDC_NAMESPACE + "properties", createObjectBuilder() .add(EDC_NAMESPACE + "type", "AzureStorage") - .add(EDC_NAMESPACE + "account", consumerAzuriteAccount.name()) - .add(EDC_NAMESPACE + "container", CONSUMER_CONTAINER_NAME) + .add(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name()) + .add(EDC_NAMESPACE + "container", destinationContainerName) .add(EDC_NAMESPACE + "folderName", destfolder) .build()) .build(); @@ -163,7 +165,7 @@ void azureBlobPush_withDestFolder() { await().atMost(ASYNC_TIMEOUT).untilAsserted(() -> { var state = consumer().getTransferProcessState(transferProcessId); assertThat(state).isEqualTo(COMPLETED.name()); - assertThat(consumerBlobHelper.listBlobs(CONSUMER_CONTAINER_NAME)) + assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .contains("%s/folder/%s".formatted(destfolder, TESTFILE_NAME)); }); } @@ -172,21 +174,23 @@ void azureBlobPush_withDestFolder() { void azureBlobPush_withoutDestFolder() { var assetId = "felix-blob-test-asset"; + var sourceContainerName = UUID.randomUUID().toString(); Map dataAddress = Map.of( "name", "transfer-test", "@type", "DataAddress", "type", "AzureStorage", - "container", PROVIDER_CONTAINER_NAME, - "account", providerAzuriteAccount.name(), + "container", sourceContainerName, + "account", PROVIDER_AZURITE_ACCOUNT.name(), "blobPrefix", "folder/", "keyName", PROVIDER_KEY_ALIAS ); // upload file to provider's blob store - var sourceContainer = providerBlobHelper.createContainer(PROVIDER_CONTAINER_NAME); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/" + TESTFILE_NAME); - consumerBlobHelper.createContainer(CONSUMER_CONTAINER_NAME); + var destinationContainerName = UUID.randomUUID().toString(); + consumerBlobHelper.createContainer(destinationContainerName); // create objects in EDC provider().createAsset(assetId, Map.of(), dataAddress); @@ -198,8 +202,8 @@ void azureBlobPush_withoutDestFolder() { .add(EDC_NAMESPACE + "type", "AzureStorage") .add(EDC_NAMESPACE + "properties", createObjectBuilder() .add(EDC_NAMESPACE + "type", "AzureStorage") - .add(EDC_NAMESPACE + "account", consumerAzuriteAccount.name()) - .add(EDC_NAMESPACE + "container", CONSUMER_CONTAINER_NAME) + .add(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name()) + .add(EDC_NAMESPACE + "container", destinationContainerName) // .add(EDC_NAMESPACE + "folderName", destfolder) <-- no dest folder .build()) .build(); @@ -214,7 +218,7 @@ void azureBlobPush_withoutDestFolder() { await().atMost(ASYNC_TIMEOUT).untilAsserted(() -> { var state = consumer().getTransferProcessState(transferProcessId); assertThat(state).isEqualTo(COMPLETED.name()); - assertThat(consumerBlobHelper.listBlobs(CONSUMER_CONTAINER_NAME)) + assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .contains("folder/%s".formatted(TESTFILE_NAME)); }); } @@ -223,34 +227,35 @@ void azureBlobPush_withoutDestFolder() { void azureBlobPush_containerNotExist() { var assetId = "blob-test-asset"; + var sourceContainerName = UUID.randomUUID().toString(); Map dataAddress = Map.of( "name", "transfer-test", "@type", "DataAddress", "type", "AzureStorage", - "container", PROVIDER_CONTAINER_NAME, - "account", providerAzuriteAccount.name(), + "container", sourceContainerName, + "account", PROVIDER_AZURITE_ACCOUNT.name(), "blobPrefix", "folder/", "keyName", PROVIDER_KEY_ALIAS ); // upload file to provider's blob store - var sourceContainer = providerBlobHelper.createContainer(PROVIDER_CONTAINER_NAME); + var sourceContainer = providerBlobHelper.createContainer(sourceContainerName); var fileData = BinaryData.fromString(TestUtils.getResourceFileContentAsString(TESTFILE_NAME)); providerBlobHelper.uploadBlob(sourceContainer, fileData, "folder/" + TESTFILE_NAME); - // consumerBlobHelper.createContainer(CONSUMER_CONTAINER_NAME); <-- container is not created // create objects in EDC provider().createAsset(assetId, Map.of(), dataAddress); var policyId = provider().createPolicyDefinition(bnpPolicy(consumer().getBpn())); provider().createContractDefinition(assetId, "def-1", policyId, policyId); + var destinationContainerName = UUID.randomUUID().toString(); var destination = createObjectBuilder() .add(TYPE, EDC_NAMESPACE + "DataAddress") .add(EDC_NAMESPACE + "type", "AzureStorage") .add(EDC_NAMESPACE + "properties", createObjectBuilder() .add(EDC_NAMESPACE + "type", "AzureStorage") - .add(EDC_NAMESPACE + "account", consumerAzuriteAccount.name()) - .add(EDC_NAMESPACE + "container", CONSUMER_CONTAINER_NAME) + .add(EDC_NAMESPACE + "account", CONSUMER_AZURITE_ACCOUNT.name()) + .add(EDC_NAMESPACE + "container", destinationContainerName) .build()) .build(); @@ -264,7 +269,7 @@ void azureBlobPush_containerNotExist() { await().atMost(ASYNC_TIMEOUT).untilAsserted(() -> { var state = consumer().getTransferProcessState(transferProcessId); assertThat(state).isEqualTo(COMPLETED.name()); - assertThat(consumerBlobHelper.listBlobs(CONSUMER_CONTAINER_NAME)) + assertThat(consumerBlobHelper.listBlobs(destinationContainerName)) .contains("folder/%s".formatted(TESTFILE_NAME)); }); } diff --git a/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/S3ToS3EndToEndTest.java b/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/S3ToS3EndToEndTest.java index 8e70034a0..9a0d2c5cf 100644 --- a/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/S3ToS3EndToEndTest.java +++ b/edc-tests/edc-end2end/end2end-transfer-cloud/src/test/java/org/eclipse/tractusx/edc/tests/transfer/S3ToS3EndToEndTest.java @@ -26,24 +26,22 @@ import org.eclipse.edc.junit.annotations.EndToEndTest; import org.eclipse.edc.junit.extensions.RuntimeExtension; import org.eclipse.edc.junit.testfixtures.TestUtils; -import org.eclipse.tractusx.edc.tests.aws.MinioContainer; +import org.eclipse.tractusx.edc.tests.aws.MinioExtension; import org.eclipse.tractusx.edc.tests.participant.TractusxParticipantBase; import org.eclipse.tractusx.edc.tests.participant.TransferParticipant; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Object; import java.net.URI; -import java.util.List; import java.util.Map; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -77,14 +75,14 @@ public class S3ToS3EndToEndTest { protected static final RuntimeExtension PROVIDER_RUNTIME = memoryRuntime(PROVIDER.getName(), PROVIDER.getBpn(), PROVIDER.getConfiguration()); @RegisterExtension protected static final RuntimeExtension CONSUMER_RUNTIME = memoryRuntime(CONSUMER.getName(), CONSUMER.getBpn(), CONSUMER.getConfiguration()); + private static final String S3_REGION = Region.US_WEST_2.id(); - private static final String S3_PROVIDER_BUCKET_NAME = "provider-bucket"; - private static final String S3_CONSUMER_BUCKET_NAME = "consumer-bucket" + System.currentTimeMillis(); private static final String TESTFILE_NAME = "hello.txt"; - @Container - private final MinioContainer providerContainer = new MinioContainer(); - @Container - private final MinioContainer consumerContainer = new MinioContainer(); + + @RegisterExtension + private static final MinioExtension PROVIDER_CONTAINER = new MinioExtension(); + @RegisterExtension + private static final MinioExtension CONSUMER_CONTAINER = new MinioExtension(); private S3Client providerClient; private S3Client consumerClient; @@ -93,17 +91,17 @@ public class S3ToS3EndToEndTest { @BeforeEach void setup() { - providerEndpointOverride = "http://localhost:%s/".formatted(providerContainer.getFirstMappedPort()); + providerEndpointOverride = "http://localhost:%s/".formatted(PROVIDER_CONTAINER.getPort()); var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(providerEndpointOverride)) - .credentialsProvider(providerContainer::getCredentials) + .credentialsProvider(PROVIDER_CONTAINER::getCredentials) .build(); providerClient = new AwsClientProviderImpl(providerConfig).s3Client(S3ClientRequest.from(S3_REGION, providerEndpointOverride)); - consumerEndpointOverride = "http://localhost:%s".formatted(consumerContainer.getFirstMappedPort()); + consumerEndpointOverride = "http://localhost:%s".formatted(CONSUMER_CONTAINER.getPort()); var consumerConfig = AwsClientProviderConfiguration.Builder.newInstance() .endpointOverride(URI.create(consumerEndpointOverride)) - .credentialsProvider(consumerContainer::getCredentials) + .credentialsProvider(CONSUMER_CONTAINER::getCredentials) .build(); consumerClient = new AwsClientProviderImpl(consumerConfig).s3Client(S3ClientRequest.from(S3_REGION, consumerEndpointOverride)); } @@ -113,14 +111,16 @@ void transferFile_success() { var assetId = "s3-test-asset"; // create bucket in provider - var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + var sourceBucketName = UUID.randomUUID().toString(); + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(sourceBucketName).build()); assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); // upload test file in provider - var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(sourceBucketName).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); // create bucket in consumer - var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + var destinationBucketName = UUID.randomUUID().toString(); + var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(destinationBucketName).build()); assertThat(b2.sdkHttpResponse().isSuccessful()).isTrue(); Map dataAddress = Map.of( @@ -129,9 +129,9 @@ void transferFile_success() { "type", "AmazonS3", "objectName", TESTFILE_NAME, "region", S3_REGION, - "bucketName", S3_PROVIDER_BUCKET_NAME, - "accessKeyId", providerContainer.getCredentials().accessKeyId(), - "secretAccessKey", providerContainer.getCredentials().secretAccessKey(), + "bucketName", sourceBucketName, + "accessKeyId", PROVIDER_CONTAINER.getCredentials().accessKeyId(), + "secretAccessKey", PROVIDER_CONTAINER.getCredentials().secretAccessKey(), "endpointOverride", providerEndpointOverride ); @@ -148,10 +148,10 @@ void transferFile_success() { .add(EDC_NAMESPACE + "type", "AmazonS3") .add(EDC_NAMESPACE + "objectName", TESTFILE_NAME) .add(EDC_NAMESPACE + "region", S3_REGION) - .add(EDC_NAMESPACE + "bucketName", S3_CONSUMER_BUCKET_NAME) + .add(EDC_NAMESPACE + "bucketName", destinationBucketName) .add(EDC_NAMESPACE + "endpointOverride", consumerEndpointOverride) - .add(EDC_NAMESPACE + "accessKeyId", consumerContainer.getCredentials().accessKeyId()) - .add(EDC_NAMESPACE + "secretAccessKey", consumerContainer.getCredentials().secretAccessKey()) + .add(EDC_NAMESPACE + "accessKeyId", CONSUMER_CONTAINER.getCredentials().accessKeyId()) + .add(EDC_NAMESPACE + "secretAccessKey", CONSUMER_CONTAINER.getCredentials().secretAccessKey()) .build() ).build(); @@ -164,7 +164,7 @@ void transferFile_success() { await().atMost(ASYNC_TIMEOUT).untilAsserted(() -> { var state = consumer().getTransferProcessState(transferProcessId); assertThat(state).isEqualTo(COMPLETED.name()); - var rq = ListObjectsRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build(); + var rq = ListObjectsRequest.builder().bucket(destinationBucketName).build(); assertThat(consumerClient.listObjects(rq).contents()).isNotEmpty(); }); } @@ -177,9 +177,4 @@ public TractusxParticipantBase consumer() { return CONSUMER; } - private List listObjects(S3Client consumerClient, String bucketName) { - var response = consumerClient.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()); - return response.contents().stream().map(S3Object::key).toList(); - } - }