From f097532f944644a599782ee4ef64e0e98a7822ca Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Thu, 27 Jan 2022 15:17:53 +0100 Subject: [PATCH 01/35] Add docker file for sample 04 --- Dockerfile | 20 +++++++++++++++++++ docker-compose.yml | 18 +++++++++++++++++ .../inline/core/InlineDataFlowController.java | 15 +++++++------- 3 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..023b00be6af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Dependecies +FROM gradle:jdk11-alpine AS gradle + +WORKDIR /app + +COPY ./* . + +RUN ./gradlew samples:04.0-file-transfer:consumer:build +RUN ./gradlew samples:04.0-file-transfer:provider:build + + +# FROM eclipse-temurin:11-jre-alpine AS runtime + +FROM gradle AS connector-provider + +ENTRYPOINT ["java" ,"-Dedc.fs.config=samples/04.0-file-transfer/provider/config.properties", "-jar", "samples/04.0-file-transfer/provider/build/libs/provider.jar"] + +FROM gradle AS connector-consumer + +ENTRYPOINT ["java" ,"-Dedc.fs.config=samples/04.0-file-transfer/consumer/config.properties", "-jar", "samples/04.0-file-transfer/consumer/build/libs/consumer.jar"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000000..d3293a7a2e0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3.8' + +services: + connector-provider: + container_name: sample04-provider + build: + context: . + target: connector-provider + ports: + - "8181:8181" + + connector-consumer: + container_name: sample04-consumer + build: + context: . + target: connector-consumer + ports: + - "9191:9191" diff --git a/extensions/inline-data-transfer/inline-data-transfer-core/src/main/java/org/eclipse/dataspaceconnector/transfer/inline/core/InlineDataFlowController.java b/extensions/inline-data-transfer/inline-data-transfer-core/src/main/java/org/eclipse/dataspaceconnector/transfer/inline/core/InlineDataFlowController.java index 71e87486eca..bb2f4eec74a 100644 --- a/extensions/inline-data-transfer/inline-data-transfer-core/src/main/java/org/eclipse/dataspaceconnector/transfer/inline/core/InlineDataFlowController.java +++ b/extensions/inline-data-transfer/inline-data-transfer-core/src/main/java/org/eclipse/dataspaceconnector/transfer/inline/core/InlineDataFlowController.java @@ -56,14 +56,6 @@ public boolean canHandle(DataRequest dataRequest) { var destinationType = dataRequest.getDestinationType(); monitor.info(format("Copying data from %s to %s", source.getType(), destinationType)); - var destSecretName = dataRequest.getDataDestination().getKeyName(); - if (destSecretName == null) { - monitor.severe(format("No credentials found for %s, will not copy!", destinationType)); - return DataFlowInitiateResult.failure(ERROR_RETRY, "Did not find credentials for data destination."); - } - - var secret = vault.resolveSecret(destSecretName); - // first look for a streamer DataStreamPublisher streamer = dataOperatorRegistry.getStreamPublisher(dataRequest); if (streamer != null) { @@ -72,6 +64,13 @@ public boolean canHandle(DataRequest dataRequest) { return DataFlowInitiateResult.failure(ERROR_RETRY, "Failed to copy data from source to destination: " + copyResult.getFailure().getMessages()); } } else { + var destSecretName = dataRequest.getDataDestination().getKeyName(); + if (destSecretName == null) { + monitor.severe(format("No credentials found for %s, will not copy!", destinationType)); + return DataFlowInitiateResult.failure(ERROR_RETRY, "Did not find credentials for data destination."); + } + + var secret = vault.resolveSecret(destSecretName); // if no copier found for this source/destination pair, then use inline read and write var reader = dataOperatorRegistry.getReader(source.getType()); var writer = dataOperatorRegistry.getWriter(destinationType); From 22263ff81ab69fc4c37cdc813ce211a28411fa6e Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Thu, 27 Jan 2022 15:27:25 +0100 Subject: [PATCH 02/35] Update Dockerfile --- Dockerfile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 023b00be6af..0cee3bc79a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,18 @@ # Dependecies -FROM gradle:jdk11-alpine AS gradle +#FROM gradle:jdk11-alpine AS gradle +FROM eclipse-temurin:11-jdk-alpine AS runtime WORKDIR /app -COPY ./* . +COPY ./ ./ -RUN ./gradlew samples:04.0-file-transfer:consumer:build -RUN ./gradlew samples:04.0-file-transfer:provider:build +RUN ["./gradlew", "samples:04.0-file-transfer:consumer:build"] +RUN ["./gradlew", "samples:04.0-file-transfer:provider:build"] +FROM runtime AS connector-provider -# FROM eclipse-temurin:11-jre-alpine AS runtime +ENTRYPOINT ["java", "-Dedc.fs.config=samples/04.0-file-transfer/provider/config.properties", "-jar", "samples/04.0-file-transfer/provider/build/libs/provider.jar"] -FROM gradle AS connector-provider +FROM runtime AS connector-consumer -ENTRYPOINT ["java" ,"-Dedc.fs.config=samples/04.0-file-transfer/provider/config.properties", "-jar", "samples/04.0-file-transfer/provider/build/libs/provider.jar"] - -FROM gradle AS connector-consumer - -ENTRYPOINT ["java" ,"-Dedc.fs.config=samples/04.0-file-transfer/consumer/config.properties", "-jar", "samples/04.0-file-transfer/consumer/build/libs/consumer.jar"] +ENTRYPOINT ["java", "-Dedc.fs.config=samples/04.0-file-transfer/consumer/config.properties", "-jar", "samples/04.0-file-transfer/consumer/build/libs/consumer.jar"] From a4c387dac0092c7a7a91672b3058eeb4ab141f96 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 28 Jan 2022 08:20:36 +0100 Subject: [PATCH 03/35] Add integration test project for samples --- samples/integration-tests/build.gradle.kts | 19 +++++++++++++++++++ .../samples/FileTransferIntegrationTest.java | 4 ++++ settings.gradle.kts | 1 + 3 files changed, 24 insertions(+) create mode 100644 samples/integration-tests/build.gradle.kts create mode 100644 samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java diff --git a/samples/integration-tests/build.gradle.kts b/samples/integration-tests/build.gradle.kts new file mode 100644 index 00000000000..509ffae8d34 --- /dev/null +++ b/samples/integration-tests/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + java +} + +group = "org.eclipse.dataspaceconnector" +version = "0.0.1-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation("org.junit.jupiter:junit-jupiter-api:5.6.0") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") +} + +tasks.getByName("test") { + useJUnitPlatform() +} \ No newline at end of file diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java new file mode 100644 index 00000000000..c33d1e902ab --- /dev/null +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java @@ -0,0 +1,4 @@ +package org.eclipse.dataspaceconnector.samples; + +public class FileTransferIntegrationTest { +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 444b860c8ca..536834f258e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -152,3 +152,4 @@ include(":samples:05-file-transfer-cloud:provider") include(":samples:05-file-transfer-cloud:api") include(":samples:05-file-transfer-cloud:data-seeder") include(":samples:05-file-transfer-cloud:transfer-file") +include(":samples:integration-tests") From 70aac69a0ec4f5f8872f919e6d5d73ffaa5d3f7a Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:30:59 +0100 Subject: [PATCH 04/35] Add first integration test for sample 04 --- samples/integration-tests/build.gradle.kts | 25 +++++++-- .../samples/FileTransferIntegrationTest.java | 56 +++++++++++++++++++ .../src/test/resources/contractoffer.json | 45 +++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 samples/integration-tests/src/test/resources/contractoffer.json diff --git a/samples/integration-tests/build.gradle.kts b/samples/integration-tests/build.gradle.kts index 509ffae8d34..b623fcf9573 100644 --- a/samples/integration-tests/build.gradle.kts +++ b/samples/integration-tests/build.gradle.kts @@ -1,17 +1,32 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + plugins { java } -group = "org.eclipse.dataspaceconnector" -version = "0.0.1-SNAPSHOT" - repositories { mavenCentral() } dependencies { - testImplementation("org.junit.jupiter:junit-jupiter-api:5.6.0") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.2") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.2") + testImplementation("io.rest-assured:rest-assured:4.5.0") + testImplementation("org.assertj:assertj-core:3.22.0") + + testImplementation(testFixtures(project(":common:util"))) } tasks.getByName("test") { diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java index c33d1e902ab..fca687306f8 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java @@ -1,4 +1,60 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + package org.eclipse.dataspaceconnector.samples; +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; +import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static io.restassured.RestAssured.given; +import static java.lang.String.format; +import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; + +/** + * Integration Test for Sample 04.0-file-transfer + */ +@IntegrationTest public class FileTransferIntegrationTest { + + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; + private static final String CONNECTOR_ADDRESS = "connectorAddress"; + private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + + + @BeforeAll + static void setUp() { + RestAssured.baseURI = CONSUMER_CONNECTOR_HOST; + } + + @Test + public void negotiateContract_success() { + var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + + var response = + given() + .contentType(ContentType.JSON) + .queryParam(CONNECTOR_ADDRESS, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .body(contractOffer) + .when() + .post(CONTRACT_NEGOTIATION_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK); + } } diff --git a/samples/integration-tests/src/test/resources/contractoffer.json b/samples/integration-tests/src/test/resources/contractoffer.json new file mode 100644 index 00000000000..9c7ddc1683a --- /dev/null +++ b/samples/integration-tests/src/test/resources/contractoffer.json @@ -0,0 +1,45 @@ +{ + "id": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "policy": { + "uid": "956e172f-2de1-4501-8881-057a57fd0e69", + "permissions": [ + { + "edctype": "dataspaceconnector:permission", + "uid": null, + "target": "test-document", + "action": { + "type": "USE", + "includedIn": null, + "constraint": null + }, + "assignee": null, + "assigner": null, + "constraints": [], + "duties": [] + } + ], + "prohibitions": [], + "obligations": [], + "extensibleProperties": {}, + "inheritsFrom": null, + "assigner": null, + "assignee": null, + "target": null, + "@type": { + "@policytype": "set" + } + }, + "asset": { + "properties": { + "ids:byteSize": null, + "asset:prop:id": "test-document", + "ids:fileName": null + } + }, + "provider": null, + "consumer": null, + "offerStart": null, + "offerEnd": null, + "contractStart": null, + "contractEnd": null +} From 24a8c2bb2d3502f13e091618f5a15b09e32b4c6a Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 28 Jan 2022 13:24:29 +0100 Subject: [PATCH 05/35] Update FileTransferSystemTest --- samples/integration-tests/build.gradle.kts | 2 + .../samples/FileTransferIntegrationTest.java | 60 ---------- .../samples/FileTransferSystemTest.java | 109 ++++++++++++++++++ 3 files changed, 111 insertions(+), 60 deletions(-) delete mode 100644 samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java create mode 100644 samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java diff --git a/samples/integration-tests/build.gradle.kts b/samples/integration-tests/build.gradle.kts index b623fcf9573..f41ca6966ab 100644 --- a/samples/integration-tests/build.gradle.kts +++ b/samples/integration-tests/build.gradle.kts @@ -25,6 +25,8 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.2") testImplementation("io.rest-assured:rest-assured:4.5.0") testImplementation("org.assertj:assertj-core:3.22.0") + testImplementation("org.awaitility:awaitility:4.1.1") + testImplementation("net.javacrumbs.json-unit:json-unit-assertj:2.28.0") testImplementation(testFixtures(project(":common:util"))) } diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java deleted file mode 100644 index fca687306f8..00000000000 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferIntegrationTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.dataspaceconnector.samples; - -import io.restassured.RestAssured; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; -import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import static io.restassured.RestAssured.given; -import static java.lang.String.format; -import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; - -/** - * Integration Test for Sample 04.0-file-transfer - */ -@IntegrationTest -public class FileTransferIntegrationTest { - - private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; - private static final String CONNECTOR_ADDRESS = "connectorAddress"; - private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); - private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); - - - @BeforeAll - static void setUp() { - RestAssured.baseURI = CONSUMER_CONNECTOR_HOST; - } - - @Test - public void negotiateContract_success() { - var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); - - var response = - given() - .contentType(ContentType.JSON) - .queryParam(CONNECTOR_ADDRESS, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) - .body(contractOffer) - .when() - .post(CONTRACT_NEGOTIATION_PATH) - .then() - .assertThat().statusCode(HttpStatus.SC_OK); - } -} diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java new file mode 100644 index 00000000000..9e7dbd070a8 --- /dev/null +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.dataspaceconnector.samples; + +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; +import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.Callable; +import java.util.function.Predicate; + +import static io.restassured.RestAssured.given; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.json; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; + +/** + * System Test for Sample 04.0-file-transfer + */ +@Tag("SystemTests") +@IntegrationTest +public class FileTransferSystemTest { + + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; + private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; + private static final String CONNECTOR_ADDRESS = "connectorAddress"; + private static final String CONTRACT_NEGOTIATION_REQUEST_ID = "contractNegotiationRequestId"; + private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); + private static final String API_KEY_HEADER = "X-Api-Key"; + + @BeforeAll + static void setUp() { + RestAssured.baseURI = CONSUMER_CONNECTOR_HOST; + } + + @Test + public void transferFile_success() { + //Arrange + var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + + //Act + + // Initiate a contract negotiation + var contractNegotiationRequestId = + given() + .contentType(ContentType.JSON) + .queryParam(CONNECTOR_ADDRESS, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .body(contractOffer) + .when() + .post(CONTRACT_NEGOTIATION_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // UUID is returned to get the contract agreement negotiated between provider and consumer. + assertThat(contractNegotiationRequestId).isNotBlank(); + + // Verify ContractNegotiation is CONFIRMED (state = 1200) + await().atMost(60, SECONDS).untilAsserted(() -> { + + assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId)).and( + json -> json.node("id").isEqualTo(contractNegotiationRequestId), + json -> json.node("state").isEqualTo(1200), + json -> json.node("contractAgreement.id").isNotNull() + ); + }); + +// //Get contract agreement id. +// var contractAgreementId = assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId)) +// .node("contractAgreement.id").asString(); + + + } + + private String getNegotiatedAgreement(String contractNegotiationRequestId) { + return + given() + .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID, contractNegotiationRequestId) + .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) + .when() + .get(CONTRACT_AGREEMENT_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + } +} From c9f54b8709e97307fe5462d2fd96c0467815e746 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Sat, 29 Jan 2022 22:19:31 +0100 Subject: [PATCH 06/35] Update sample04 system test --- .../samples/FileTransferSystemTest.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index 9e7dbd070a8..823b52a1919 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -14,6 +14,7 @@ package org.eclipse.dataspaceconnector.samples; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.restassured.RestAssured; import io.restassured.http.ContentType; import org.apache.http.HttpStatus; @@ -23,16 +24,12 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.util.concurrent.Callable; -import java.util.function.Predicate; - import static io.restassured.RestAssured.given; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; -import static net.javacrumbs.jsonunit.assertj.JsonAssertions.json; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; -import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; /** @@ -79,23 +76,23 @@ public void transferFile_success() { assertThat(contractNegotiationRequestId).isNotBlank(); // Verify ContractNegotiation is CONFIRMED (state = 1200) - await().atMost(60, SECONDS).untilAsserted(() -> { + await().atMost(30, SECONDS).untilAsserted(() -> { - assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId)).and( + assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId).toString()).and( json -> json.node("id").isEqualTo(contractNegotiationRequestId), json -> json.node("state").isEqualTo(1200), json -> json.node("contractAgreement.id").isNotNull() ); }); -// //Get contract agreement id. -// var contractAgreementId = assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId)) -// .node("contractAgreement.id").asString(); - + // Obtain contract agreement ID + var contractAgreementId = getNegotiatedAgreement(contractNegotiationRequestId) + .get("contractAgreement").get("id").toString(); + assertThat(contractAgreementId).isNotNull(); } - private String getNegotiatedAgreement(String contractNegotiationRequestId) { + private ObjectNode getNegotiatedAgreement(String contractNegotiationRequestId) { return given() .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID, contractNegotiationRequestId) @@ -104,6 +101,6 @@ private String getNegotiatedAgreement(String contractNegotiationRequestId) { .get(CONTRACT_AGREEMENT_PATH) .then() .assertThat().statusCode(HttpStatus.SC_OK) - .extract().asString(); + .extract().as(ObjectNode.class); } } From 89b1d5d5b002b73736d77816e7fd608cbd90c3d3 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Mon, 31 Jan 2022 09:58:24 +0100 Subject: [PATCH 07/35] Update sample 04 test with initiate a file transfer --- .../samples/FileTransferSystemTest.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index 823b52a1919..e35d16e1641 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -41,10 +41,19 @@ public class FileTransferSystemTest { private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; - private static final String CONNECTOR_ADDRESS = "connectorAddress"; - private static final String CONTRACT_NEGOTIATION_REQUEST_ID = "contractNegotiationRequestId"; + private static final String FILE_TRANSFER_PATH = "/api/file/test-document"; + + private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; + private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; + private static final String DESTINATION_PARAM = "destination"; + private static final String CONTRACT_ID_PARAM = "contractId"; + private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); + private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", "/tmp/provider/test-document.txt"); + private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); private static final String API_KEY_HEADER = "X-Api-Key"; @@ -58,13 +67,13 @@ public void transferFile_success() { //Arrange var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); - //Act + //Act & Assert // Initiate a contract negotiation var contractNegotiationRequestId = given() .contentType(ContentType.JSON) - .queryParam(CONNECTOR_ADDRESS, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) .body(contractOffer) .when() .post(CONTRACT_NEGOTIATION_PATH) @@ -87,15 +96,28 @@ public void transferFile_success() { // Obtain contract agreement ID var contractAgreementId = getNegotiatedAgreement(contractNegotiationRequestId) - .get("contractAgreement").get("id").toString(); + .get("contractAgreement").get("id").textValue(); + + //Initiate file transfer + var transferProcessId = + given() + .noContentType() + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) + .queryParam(CONTRACT_ID_PARAM, contractAgreementId) + .when() + .post(FILE_TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); - assertThat(contractAgreementId).isNotNull(); + assertThat(transferProcessId).isNotNull(); } private ObjectNode getNegotiatedAgreement(String contractNegotiationRequestId) { return given() - .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID, contractNegotiationRequestId) + .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) .when() .get(CONTRACT_AGREEMENT_PATH) From 367c51b104c2ac7af2e6f4488759f34c99f60986 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Mon, 31 Jan 2022 13:59:39 +0100 Subject: [PATCH 08/35] Updated Sample04 file transfer test --- .../samples/FileTransferSystemTest.java | 63 +++++++++++++++---- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index e35d16e1641..d78241b5a3c 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -24,6 +24,12 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + import static io.restassured.RestAssured.given; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; @@ -39,20 +45,23 @@ @IntegrationTest public class FileTransferSystemTest { + private static final String PROVIDER_ASSET_NAME = "test-document"; + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; - private static final String FILE_TRANSFER_PATH = "/api/file/test-document"; + private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; private static final String DESTINATION_PARAM = "destination"; private static final String CONTRACT_ID_PARAM = "contractId"; + private static final String FILE_NAME_PARAM = "filename"; private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); - private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", "/tmp/provider/test-document.txt"); + private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); private static final String API_KEY_HEADER = "X-Api-Key"; @@ -63,9 +72,12 @@ static void setUp() { } @Test - public void transferFile_success() { + public void transferFile_success() throws IOException { //Arrange var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + //Create a file with test data on provide file system. + var fileContent = "Sample04-test-" + UUID.randomUUID(); + Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); //Act & Assert @@ -75,9 +87,9 @@ public void transferFile_success() { .contentType(ContentType.JSON) .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) .body(contractOffer) - .when() + .when() .post(CONTRACT_NEGOTIATION_PATH) - .then() + .then() .assertThat().statusCode(HttpStatus.SC_OK) .extract().asString(); @@ -87,7 +99,7 @@ public void transferFile_success() { // Verify ContractNegotiation is CONFIRMED (state = 1200) await().atMost(30, SECONDS).untilAsserted(() -> { - assertThatJson(getNegotiatedAgreement(contractNegotiationRequestId).toString()).and( + assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( json -> json.node("id").isEqualTo(contractNegotiationRequestId), json -> json.node("state").isEqualTo(1200), json -> json.node("contractAgreement.id").isNotNull() @@ -95,34 +107,61 @@ public void transferFile_success() { }); // Obtain contract agreement ID - var contractAgreementId = getNegotiatedAgreement(contractNegotiationRequestId) + var contractAgreementId = fetchNegotiatedAgreement(contractNegotiationRequestId) .get("contractAgreement").get("id").textValue(); //Initiate file transfer var transferProcessId = given() .noContentType() + .pathParam(FILE_NAME_PARAM, PROVIDER_ASSET_NAME) .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) .queryParam(CONTRACT_ID_PARAM, contractAgreementId) - .when() + .when() .post(FILE_TRANSFER_PATH) - .then() + .then() .assertThat().statusCode(HttpStatus.SC_OK) .extract().asString(); + // Verify TransferProcessId assertThat(transferProcessId).isNotNull(); + + //Verify file transfer is completed and file contents + await().atMost(30, SECONDS).untilAsserted(() -> { + + var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); + var actualFileContent = fetchFileContent(copiedFilePath); + assertThat(actualFileContent).isNotNull(); + assertThat(actualFileContent).isEqualTo(fileContent); + }); } - private ObjectNode getNegotiatedAgreement(String contractNegotiationRequestId) { + /** + * Fetch negotiated contract agreement. + * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. + * @return Negotiation as {@link ObjectNode}. + */ + private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) { return given() .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) - .when() + .when() .get(CONTRACT_AGREEMENT_PATH) - .then() + .then() .assertThat().statusCode(HttpStatus.SC_OK) .extract().as(ObjectNode.class); } + + private String fetchFileContent(Path filePath) { + if (filePath.toFile().exists()) { + try { + return Files.readAllLines(filePath).get(0); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } } From e64d92cf3a15c8380a6175d6ec2d7d79fd444222 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Tue, 1 Feb 2022 08:09:56 +0100 Subject: [PATCH 09/35] Remove unused docker files --- Dockerfile | 18 ------------------ docker-compose.yml | 18 ------------------ 2 files changed, 36 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0cee3bc79a7..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Dependecies -#FROM gradle:jdk11-alpine AS gradle -FROM eclipse-temurin:11-jdk-alpine AS runtime - -WORKDIR /app - -COPY ./ ./ - -RUN ["./gradlew", "samples:04.0-file-transfer:consumer:build"] -RUN ["./gradlew", "samples:04.0-file-transfer:provider:build"] - -FROM runtime AS connector-provider - -ENTRYPOINT ["java", "-Dedc.fs.config=samples/04.0-file-transfer/provider/config.properties", "-jar", "samples/04.0-file-transfer/provider/build/libs/provider.jar"] - -FROM runtime AS connector-consumer - -ENTRYPOINT ["java", "-Dedc.fs.config=samples/04.0-file-transfer/consumer/config.properties", "-jar", "samples/04.0-file-transfer/consumer/build/libs/consumer.jar"] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index d3293a7a2e0..00000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' - -services: - connector-provider: - container_name: sample04-provider - build: - context: . - target: connector-provider - ports: - - "8181:8181" - - connector-consumer: - container_name: sample04-consumer - build: - context: . - target: connector-consumer - ports: - - "9191:9191" From 04abb7ee31e19d56867c385aa18200c2ef566b08 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:11:22 +0100 Subject: [PATCH 10/35] Use Enum for Test --- .../dataspaceconnector/samples/FileTransferSystemTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index d78241b5a3c..8670f472e40 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -20,6 +20,7 @@ import org.apache.http.HttpStatus; import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -101,7 +102,7 @@ public void transferFile_success() throws IOException { assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( json -> json.node("id").isEqualTo(contractNegotiationRequestId), - json -> json.node("state").isEqualTo(1200), + json -> json.node("state").isEqualTo(ContractNegotiationStates.CONFIRMED.code()), json -> json.node("contractAgreement.id").isNotNull() ); }); From 3c24ace017f5d985e8a454bbb8c2dbe327112af6 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 12:30:23 +0100 Subject: [PATCH 11/35] Move integration test module to sample04 --- samples/04.0-file-transfer/README.md | 2 +- .../integration-tests/build.gradle.kts | 3 +- .../samples/FileTransferSystemTest.java | 8 +++- .../src/test/resources}/contractoffer.json | 0 .../src/test/resources/contractoffer.json | 45 ------------------- settings.gradle.kts | 2 +- 6 files changed, 10 insertions(+), 50 deletions(-) rename samples/{ => 04.0-file-transfer}/integration-tests/build.gradle.kts (93%) rename samples/{ => 04.0-file-transfer}/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java (95%) rename samples/04.0-file-transfer/{ => integration-tests/src/test/resources}/contractoffer.json (100%) delete mode 100644 samples/integration-tests/src/test/resources/contractoffer.json diff --git a/samples/04.0-file-transfer/README.md b/samples/04.0-file-transfer/README.md index 01a32107d85..5ec880cc16f 100644 --- a/samples/04.0-file-transfer/README.md +++ b/samples/04.0-file-transfer/README.md @@ -194,7 +194,7 @@ addition to just confirming or declining an offer. In order to trigger the negotiation, we use the endpoint previously created in the `api` extension. We specify the address of the provider connector as a query parameter and set our contract offer in the request body. The contract -offer is prepared in [contractoffer.json](contractoffer.json) +offer is prepared in [contractoffer.json](integration-tests/src/test/resources/contractoffer.json) and can be used as is. In a real scenario, a potential consumer would first need to request a description of the provider's offers in order to get the provider's contract offer. diff --git a/samples/integration-tests/build.gradle.kts b/samples/04.0-file-transfer/integration-tests/build.gradle.kts similarity index 93% rename from samples/integration-tests/build.gradle.kts rename to samples/04.0-file-transfer/integration-tests/build.gradle.kts index f41ca6966ab..861a8e180fc 100644 --- a/samples/integration-tests/build.gradle.kts +++ b/samples/04.0-file-transfer/integration-tests/build.gradle.kts @@ -29,8 +29,9 @@ dependencies { testImplementation("net.javacrumbs.json-unit:json-unit-assertj:2.28.0") testImplementation(testFixtures(project(":common:util"))) + testImplementation(testFixtures(project(":launchers:junit"))) } tasks.getByName("test") { useJUnitPlatform() -} \ No newline at end of file +} diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java similarity index 95% rename from samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java rename to samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index 8670f472e40..fb242c48f60 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -133,8 +133,12 @@ public void transferFile_success() throws IOException { var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); var actualFileContent = fetchFileContent(copiedFilePath); - assertThat(actualFileContent).isNotNull(); - assertThat(actualFileContent).isEqualTo(fileContent); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are null") + .isNotNull(); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are not same as the source file") + .isEqualTo(fileContent); }); } diff --git a/samples/04.0-file-transfer/contractoffer.json b/samples/04.0-file-transfer/integration-tests/src/test/resources/contractoffer.json similarity index 100% rename from samples/04.0-file-transfer/contractoffer.json rename to samples/04.0-file-transfer/integration-tests/src/test/resources/contractoffer.json diff --git a/samples/integration-tests/src/test/resources/contractoffer.json b/samples/integration-tests/src/test/resources/contractoffer.json deleted file mode 100644 index 9c7ddc1683a..00000000000 --- a/samples/integration-tests/src/test/resources/contractoffer.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "id": "1:3a75736e-001d-4364-8bd4-9888490edb58", - "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } - }, - "provider": null, - "consumer": null, - "offerStart": null, - "offerEnd": null, - "contractStart": null, - "contractEnd": null -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 536834f258e..925fde64e63 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -152,4 +152,4 @@ include(":samples:05-file-transfer-cloud:provider") include(":samples:05-file-transfer-cloud:api") include(":samples:05-file-transfer-cloud:data-seeder") include(":samples:05-file-transfer-cloud:transfer-file") -include(":samples:integration-tests") +include("samples:04.0-file-transfer:integration-tests") From 2096831ae0ecf8b7cd303ad2d963c3da878c5c99 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 12:43:57 +0100 Subject: [PATCH 12/35] Update sample 4 integration test with error message and java doc --- .../samples/FileTransferSystemTest.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index fb242c48f60..8db8b2d43e6 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -95,15 +95,22 @@ public void transferFile_success() throws IOException { .extract().asString(); // UUID is returned to get the contract agreement negotiated between provider and consumer. - assertThat(contractNegotiationRequestId).isNotBlank(); + assertThat(contractNegotiationRequestId) + .withFailMessage("Contract negotiation requestId is null").isNotBlank(); // Verify ContractNegotiation is CONFIRMED (state = 1200) await().atMost(30, SECONDS).untilAsserted(() -> { assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( - json -> json.node("id").isEqualTo(contractNegotiationRequestId), - json -> json.node("state").isEqualTo(ContractNegotiationStates.CONFIRMED.code()), - json -> json.node("contractAgreement.id").isNotNull() + json -> json.node("id") + .withFailMessage("Negotiation id is null") + .isEqualTo(contractNegotiationRequestId), + json -> json.node("state") + .withFailMessage("ContractNegotiation is not in CONFIRMED state.") + .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), + json -> json.node("contractAgreement.id") + .withFailMessage("contractAgreement.id is null") + .isNotNull() ); }); @@ -159,6 +166,11 @@ private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) .extract().as(ObjectNode.class); } + /** + * Helper method to read file contents on the given {@link Path} + * @param filePath see {@link Path} + * @return Contents of file as a {@link String} or null if file does not exist. + */ private String fetchFileContent(Path filePath) { if (filePath.toFile().exists()) { try { From b4042d2437790494edcb4cb96639e4ecee209693 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Wed, 2 Feb 2022 14:40:19 +0100 Subject: [PATCH 13/35] Added endpoint to verify transfer complete --- .../extensions/api/ConsumerApiController.java | 11 ++++++++++ .../samples/FileTransferSystemTest.java | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java index 85aa1b83c5d..c52f4bd081d 100644 --- a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java +++ b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java @@ -113,5 +113,16 @@ public Response initiateTransfer(@PathParam("filename") String filename, @QueryP return result.failed() ? Response.status(400).build() : Response.ok(result.getContent()).build(); } + + @GET + @Path("transfer/{id}") + public Response getTransferById(@PathParam("id") String id) { + return Optional.ofNullable(transferProcessStore.find(id)) + .map( + v -> Response.ok(v).build() + ).orElse( + Response.status(NOT_ACCEPTABLE).build() + ); + } } diff --git a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index d78241b5a3c..33a8c108c28 100644 --- a/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -49,10 +49,12 @@ public class FileTransferSystemTest { private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; + private static final String TRANSFER_PATH = "/api/transfer/{transferId}"; private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; + private static final String TRANSFER_ID_PARAM = "transferId"; private static final String DESTINATION_PARAM = "destination"; private static final String CONTRACT_ID_PARAM = "contractId"; private static final String FILE_NAME_PARAM = "filename"; @@ -64,6 +66,7 @@ public class FileTransferSystemTest { private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); + private static final boolean CHECK_FILE = Boolean.valueOf(propOrEnv("CHECK_FILE", "true")); private static final String API_KEY_HEADER = "X-Api-Key"; @BeforeAll @@ -129,7 +132,13 @@ public void transferFile_success() throws IOException { //Verify file transfer is completed and file contents await().atMost(30, SECONDS).untilAsserted(() -> { + assertThatJson(fetchTransfer(transferProcessId).toString()).and( + json -> json.node("id").isEqualTo(transferProcessId), + json -> json.node("state").isEqualTo(800) // COMPLETED + ); + }); + if (CHECK_FILE) { var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); var actualFileContent = fetchFileContent(copiedFilePath); assertThat(actualFileContent).isNotNull(); @@ -137,6 +146,17 @@ public void transferFile_success() throws IOException { }); } + private ObjectNode fetchTransfer(String transferProcessId) { + return + given() + .pathParam(TRANSFER_ID_PARAM, transferProcessId) + .when() + .get(TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + /** * Fetch negotiated contract agreement. * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. From 767d2e303a00d957b3fd85a5446eb76ca7609f3c Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 18:03:01 +0100 Subject: [PATCH 14/35] Update sample 04 integration tests --- samples/04.0-file-transfer/README.md | 2 +- .../dataspaceconnector/samples/FileTransferSystemTest.java | 2 +- settings.gradle.kts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/04.0-file-transfer/README.md b/samples/04.0-file-transfer/README.md index 5ec880cc16f..e4e29163df4 100644 --- a/samples/04.0-file-transfer/README.md +++ b/samples/04.0-file-transfer/README.md @@ -199,7 +199,7 @@ and can be used as is. In a real scenario, a potential consumer would first need provider's offers in order to get the provider's contract offer. ```bash -curl -X POST -H "Content-Type: application/json" -d @samples/04-file-transfer/contractoffer.json "http://localhost:9191/api/negotiation?connectorAddress=http://localhost:8181/api/ids/multipart" +curl -X POST -H "Content-Type: application/json" -d @samples/04-file-transfer/integration-tests/src/test/resources/contractoffer.json "http://localhost:9191/api/negotiation?connectorAddress=http://localhost:8181/api/ids/multipart" ``` In the response we'll get a UUID that we can use to get the contract agreement negotiated between provider and consumer. diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index daa5c2ddd46..005c5ad27ef 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -155,7 +155,7 @@ public void transferFile_success() throws IOException { assertThat(actualFileContent) .withFailMessage("Transferred file contents are not same as the source file") .isEqualTo(fileContent); - }); + } } private ObjectNode fetchTransfer(String transferProcessId) { diff --git a/settings.gradle.kts b/settings.gradle.kts index 925fde64e63..e1136f3ae66 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -137,6 +137,7 @@ include(":samples:04.0-file-transfer:consumer") include(":samples:04.0-file-transfer:provider") include(":samples:04.0-file-transfer:api") include(":samples:04.0-file-transfer:transfer-file") +include(":samples:04.0-file-transfer:integration-tests") include(":samples:04.1-file-transfer-listener:consumer") include(":samples:04.1-file-transfer-listener:listener") @@ -152,4 +153,3 @@ include(":samples:05-file-transfer-cloud:provider") include(":samples:05-file-transfer-cloud:api") include(":samples:05-file-transfer-cloud:data-seeder") include(":samples:05-file-transfer-cloud:transfer-file") -include("samples:04.0-file-transfer:integration-tests") From 3e2da26e638a52170e40c86c025364da1d829874 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 22:34:31 +0100 Subject: [PATCH 15/35] Update Sample04 Test --- .../extensions/api/ApiEndpointExtension.java | 4 +++- .../extensions/api/ConsumerApiController.java | 11 ++++++++--- .../samples/FileTransferSystemTest.java | 3 ++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ApiEndpointExtension.java b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ApiEndpointExtension.java index a5962d9b875..ef0b054374b 100644 --- a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ApiEndpointExtension.java +++ b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ApiEndpointExtension.java @@ -19,6 +19,7 @@ import org.eclipse.dataspaceconnector.spi.system.ServiceExtension; import org.eclipse.dataspaceconnector.spi.system.ServiceExtensionContext; import org.eclipse.dataspaceconnector.spi.transfer.TransferProcessManager; +import org.eclipse.dataspaceconnector.spi.transfer.store.TransferProcessStore; public class ApiEndpointExtension implements ServiceExtension { @@ -32,6 +33,7 @@ public void initialize(ServiceExtensionContext context) { var webService = context.getService(WebService.class); var processManager = context.getService(TransferProcessManager.class); var negotiationManager = context.getService(ConsumerContractNegotiationManager.class); - webService.registerController(new ConsumerApiController(context.getMonitor(), processManager, negotiationManager)); + var transferProcessStore = context.getService(TransferProcessStore.class); + webService.registerController(new ConsumerApiController(context.getMonitor(), processManager, negotiationManager, transferProcessStore)); } } diff --git a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java index c52f4bd081d..31285ba5660 100644 --- a/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java +++ b/samples/04.0-file-transfer/api/src/main/java/org/eclipse/dataspaceconnector/extensions/api/ConsumerApiController.java @@ -27,30 +27,35 @@ import org.eclipse.dataspaceconnector.spi.contract.negotiation.response.NegotiationResult; import org.eclipse.dataspaceconnector.spi.monitor.Monitor; import org.eclipse.dataspaceconnector.spi.transfer.TransferProcessManager; +import org.eclipse.dataspaceconnector.spi.transfer.store.TransferProcessStore; import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress; import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractOfferRequest; import org.eclipse.dataspaceconnector.spi.types.domain.contract.offer.ContractOffer; import org.eclipse.dataspaceconnector.spi.types.domain.transfer.DataRequest; import java.util.Objects; +import java.util.Optional; import java.util.UUID; +import static jakarta.ws.rs.core.Response.Status.NOT_ACCEPTABLE; import static java.lang.String.format; -@Consumes({ MediaType.APPLICATION_JSON }) -@Produces({ MediaType.APPLICATION_JSON }) +@Consumes({MediaType.APPLICATION_JSON}) +@Produces({MediaType.APPLICATION_JSON}) @Path("/") public class ConsumerApiController { private final Monitor monitor; private final TransferProcessManager processManager; private final ConsumerContractNegotiationManager consumerNegotiationManager; + private final TransferProcessStore transferProcessStore; public ConsumerApiController(Monitor monitor, TransferProcessManager processManager, - ConsumerContractNegotiationManager consumerNegotiationManager) { + ConsumerContractNegotiationManager consumerNegotiationManager, TransferProcessStore transferProcessStore) { this.monitor = monitor; this.processManager = processManager; this.consumerNegotiationManager = consumerNegotiationManager; + this.transferProcessStore = transferProcessStore; } @GET diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index 005c5ad27ef..64a562a3bce 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -21,6 +21,7 @@ import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; +import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -142,7 +143,7 @@ public void transferFile_success() throws IOException { await().atMost(30, SECONDS).untilAsserted(() -> { assertThatJson(fetchTransfer(transferProcessId).toString()).and( json -> json.node("id").isEqualTo(transferProcessId), - json -> json.node("state").isEqualTo(800) // COMPLETED + json -> json.node("state").isEqualTo(TransferProcessStates.COMPLETED.code()) ); }); From 875ceefb582b3630dcaa3d310c6bb4c6ce0a6e1e Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 22:37:22 +0100 Subject: [PATCH 16/35] Update sample04 test to use test containers --- samples/04.0-file-transfer/integration-tests/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/04.0-file-transfer/integration-tests/build.gradle.kts b/samples/04.0-file-transfer/integration-tests/build.gradle.kts index 861a8e180fc..8a2d33fa3a3 100644 --- a/samples/04.0-file-transfer/integration-tests/build.gradle.kts +++ b/samples/04.0-file-transfer/integration-tests/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { testImplementation("org.assertj:assertj-core:3.22.0") testImplementation("org.awaitility:awaitility:4.1.1") testImplementation("net.javacrumbs.json-unit:json-unit-assertj:2.28.0") + testImplementation("org.testcontainers:testcontainers:1.16.3") testImplementation(testFixtures(project(":common:util"))) testImplementation(testFixtures(project(":launchers:junit"))) From 7f47112269a642bb3e31472c8664b81ccf0e2d20 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Wed, 2 Feb 2022 22:41:24 +0100 Subject: [PATCH 17/35] Update sample04 test --- .../dataspaceconnector/samples/FileTransferSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index 64a562a3bce..ffced8306fb 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -68,7 +68,7 @@ public class FileTransferSystemTest { private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); - private static final boolean CHECK_FILE = Boolean.valueOf(propOrEnv("CHECK_FILE", "true")); + private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); private static final String API_KEY_HEADER = "X-Api-Key"; @BeforeAll From 025c780f76189c91283cff741bf16e2eb996f454 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Thu, 3 Feb 2022 01:12:50 +0100 Subject: [PATCH 18/35] Update sample04 test to use test containers for consumer container --- .../integration-tests/build.gradle.kts | 2 + .../samples/FileTransferSystemTest.java | 10 +- ...leTransferWithTestContainerSystemTest.java | 240 ++++++++++++++++++ .../src/test/resources/ConsumerDockerfile | 10 + 4 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile diff --git a/samples/04.0-file-transfer/integration-tests/build.gradle.kts b/samples/04.0-file-transfer/integration-tests/build.gradle.kts index 8a2d33fa3a3..2ee99221c5d 100644 --- a/samples/04.0-file-transfer/integration-tests/build.gradle.kts +++ b/samples/04.0-file-transfer/integration-tests/build.gradle.kts @@ -31,6 +31,8 @@ dependencies { testImplementation(testFixtures(project(":common:util"))) testImplementation(testFixtures(project(":launchers:junit"))) + + implementation(project(":samples:04.0-file-transfer:provider")) } tasks.getByName("test") { diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index ffced8306fb..dc2770ea6e8 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -110,7 +110,7 @@ public void transferFile_success() throws IOException { .withFailMessage("Negotiation id is null") .isEqualTo(contractNegotiationRequestId), json -> json.node("state") - .withFailMessage("ContractNegotiation is not in CONFIRMED state.") + .withFailMessage("ContractNegotiation is not in CONFIRMED state") .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), json -> json.node("contractAgreement.id") .withFailMessage("contractAgreement.id is null") @@ -142,8 +142,12 @@ public void transferFile_success() throws IOException { //Verify file transfer is completed and file contents await().atMost(30, SECONDS).untilAsserted(() -> { assertThatJson(fetchTransfer(transferProcessId).toString()).and( - json -> json.node("id").isEqualTo(transferProcessId), - json -> json.node("state").isEqualTo(TransferProcessStates.COMPLETED.code()) + json -> json.node("id") + .withFailMessage("TransferProcessId not matched") + .isEqualTo(transferProcessId), + json -> json.node("state") + .withFailMessage("TransferProcess is not in COMPLETED state") + .isEqualTo(TransferProcessStates.COMPLETED.code()) ); }); diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java new file mode 100644 index 00000000000..3e428e2d740 --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.dataspaceconnector.samples; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; +import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; +import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; +import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; + +/** + * System Test for Sample 04.0-file-transfer + */ +@Tag("SystemTests") +@IntegrationTest +@ExtendWith(EdcExtension.class) +public class FileTransferWithTestContainerSystemTest { + + private static final String PROVIDER_ASSET_NAME = "test-document"; + + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; + private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; + private static final String TRANSFER_PATH = "/api/transfer/{transferId}"; + private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; + + private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; + private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; + private static final String TRANSFER_ID_PARAM = "transferId"; + private static final String DESTINATION_PARAM = "destination"; + private static final String CONTRACT_ID_PARAM = "contractId"; + private static final String FILE_NAME_PARAM = "filename"; + + private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); + private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); + + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); + + private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); + private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); + private static final String API_KEY_HEADER = "X-Api-Key"; + private static GenericContainer consumerContainer; + + @BeforeAll + static void setUp() { + + // Enable Testcontainers to connect to host port + Testcontainers.exposeHostPorts(8181); + + // Prepare Testcontainer of consumer connector + var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); + consumerContainer = new GenericContainer( + new ImageFromDockerfile() + .withFileFromClasspath("Dockerfile", "ConsumerDockerfile") + .withFileFromPath("consumer.jar", Path.of(rootProject + "/consumer/build/libs/consumer.jar")) + .withFileFromPath("config.properties", Path.of(rootProject + "/consumer/config.properties"))) + .withExposedPorts(9191) + .withAccessToHost(true); + + // Start consumer connector test container + consumerContainer.start(); + // Consumer connector host URI. + RestAssured.baseURI = format("http://%s:%s", consumerContainer.getHost(), consumerContainer.getFirstMappedPort()); + } + + @AfterAll + static void tearDown() { + consumerContainer.stop(); + } + + @Test + public void transferFile_success() throws IOException { + //Arrange + var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + //Create a file with test data on provide file system. + var fileContent = "Sample04-test-" + UUID.randomUUID(); + Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); + + //Act & Assert + + // Initiate a contract negotiation + var contractNegotiationRequestId = + given() + .contentType(ContentType.JSON) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .body(contractOffer) + .when() + .post(CONTRACT_NEGOTIATION_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // UUID is returned to get the contract agreement negotiated between provider and consumer. + assertThat(contractNegotiationRequestId) + .withFailMessage("Contract negotiation requestId is null").isNotBlank(); + + // Verify ContractNegotiation is CONFIRMED (state = 1200) + await().atMost(30, SECONDS).untilAsserted(() -> { + + assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( + json -> json.node("id") + .withFailMessage("Negotiation id is null") + .isEqualTo(contractNegotiationRequestId), + json -> json.node("state") + .withFailMessage("ContractNegotiation is not in CONFIRMED state.") + .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), + json -> json.node("contractAgreement.id") + .withFailMessage("contractAgreement.id is null") + .isNotNull() + ); + }); + + // Obtain contract agreement ID + var contractAgreementId = fetchNegotiatedAgreement(contractNegotiationRequestId) + .get("contractAgreement").get("id").textValue(); + + //Initiate file transfer + var transferProcessId = + given() + .noContentType() + .pathParam(FILE_NAME_PARAM, PROVIDER_ASSET_NAME) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) + .queryParam(CONTRACT_ID_PARAM, contractAgreementId) + .when() + .post(FILE_TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // Verify TransferProcessId + assertThat(transferProcessId).isNotNull(); + + //Verify file transfer is completed and file contents + await().atMost(30, SECONDS).untilAsserted(() -> { + assertThatJson(fetchTransfer(transferProcessId).toString()).and( + json -> json.node("id") + .withFailMessage("TransferProcessId not matched") + .isEqualTo(transferProcessId), + json -> json.node("state") + .withFailMessage("TransferProcess is not in COMPLETED state") + .isEqualTo(TransferProcessStates.COMPLETED.code()) + ); + }); + + if (CHECK_FILE) { + var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); + var actualFileContent = fetchFileContent(copiedFilePath); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are null") + .isNotNull(); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are not same as the source file") + .isEqualTo(fileContent); + } + } + + private ObjectNode fetchTransfer(String transferProcessId) { + return + given() + .pathParam(TRANSFER_ID_PARAM, transferProcessId) + .when() + .get(TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Fetch negotiated contract agreement. + * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. + * @return Negotiation as {@link ObjectNode}. + */ + private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) { + return + given() + .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) + .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) + .when() + .get(CONTRACT_AGREEMENT_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Helper method to read file contents on the given {@link Path} + * @param filePath see {@link Path} + * @return Contents of file as a {@link String} or null if file does not exist. + */ + private String fetchFileContent(Path filePath) { + if (filePath.toFile().exists()) { + try { + return Files.readAllLines(filePath).get(0); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} diff --git a/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile b/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile new file mode 100644 index 00000000000..c0432e2ac74 --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile @@ -0,0 +1,10 @@ +FROM openjdk:11-jre-slim + +WORKDIR /build + +COPY ./consumer.jar . +COPY ./config.properties . + +EXPOSE 9191 + +ENTRYPOINT ["java", "-Dedc.fs.config=config.properties", "-jar", "consumer.jar"] \ No newline at end of file From f19dac343d6023e52ed6c92a6024643d69dc9367 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 08:27:07 +0100 Subject: [PATCH 19/35] Show STDOUT --- samples/04.0-file-transfer/integration-tests/build.gradle.kts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/04.0-file-transfer/integration-tests/build.gradle.kts b/samples/04.0-file-transfer/integration-tests/build.gradle.kts index 2ee99221c5d..8a0ccf1359d 100644 --- a/samples/04.0-file-transfer/integration-tests/build.gradle.kts +++ b/samples/04.0-file-transfer/integration-tests/build.gradle.kts @@ -37,4 +37,8 @@ dependencies { tasks.getByName("test") { useJUnitPlatform() + testLogging { + showStandardStreams = true + events("skipped", "failed") + } } From 3a8ea245884bc29503d6f542b1e5243e394a871a Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 08:27:12 +0100 Subject: [PATCH 20/35] Create SeparateClassloaderSystemTest.java --- .../SeparateClassloaderSystemTest.java | 258 ++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java new file mode 100644 index 00000000000..ee251ceeeca --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.dataspaceconnector.samples; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; +import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.eclipse.dataspaceconnector.core.system.runtime.BaseRuntime; +import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; +import org.eclipse.dataspaceconnector.spi.system.ServiceExtension; +import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; +import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ServiceLoader; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static io.restassured.RestAssured.given; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; + +/** + * System Test for Sample 04.0-file-transfer + */ +@Tag("SystemTests") +@IntegrationTest +@ExtendWith(EdcExtension.class) +public class SeparateClassloaderSystemTest { + + private static final String PROVIDER_ASSET_NAME = "test-document"; + + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; + private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; + private static final String TRANSFER_PATH = "/api/transfer/{transferId}"; + private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; + + private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; + private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; + private static final String TRANSFER_ID_PARAM = "transferId"; + private static final String DESTINATION_PARAM = "destination"; + private static final String CONTRACT_ID_PARAM = "contractId"; + private static final String FILE_NAME_PARAM = "filename"; + + private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); + private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); + + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); + + private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); + private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); + private static final String API_KEY_HEADER = "X-Api-Key"; + + @BeforeAll + static void setUp() throws Exception { + + // Prepare Testcontainer of consumer connector + var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); + + System.setProperty("web.http.port", "9191"); + System.setProperty("edc.api.control.auth.apikey.value", "password"); + var c = new CountDownLatch(1); + Thread t = new Thread(() -> + { + ClassLoader parent = null; + parent = ClassLoader.getSystemClassLoader(); + URLClassLoader child = null; + try { + File file = new File(rootProject + + "/consumer/build/libs/consumer.jar"); + assertThat(file).canRead(); + + child = URLClassLoader.newInstance(new URL[]{file.toURL() + }, + parent); + Thread.currentThread().setContextClassLoader(child); + Class r = child.loadClass(BaseRuntime.class.getCanonicalName()); + var m = r.getMethod("main", String[].class); + m.invoke(null, new Object[]{new String[0]}); + c.countDown(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + t.start(); + c.await(10, SECONDS); + System.clearProperty("web.http.port"); + + // Consumer connector host URI. + RestAssured.baseURI = format("http://%s:%s", "localhost", 9191); + } + + @Test + public void transferFile_success() throws IOException { + //Arrange + var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + //Create a file with test data on provide file system. + var fileContent = "Sample04-test-" + UUID.randomUUID(); + Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); + + //Act & Assert + + // Initiate a contract negotiation + var contractNegotiationRequestId = + given() + .contentType(ContentType.JSON) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .body(contractOffer) + .when() + .post(CONTRACT_NEGOTIATION_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // UUID is returned to get the contract agreement negotiated between provider and consumer. + assertThat(contractNegotiationRequestId) + .withFailMessage("Contract negotiation requestId is null").isNotBlank(); + + // Verify ContractNegotiation is CONFIRMED (state = 1200) + await().atMost(30, SECONDS).untilAsserted(() -> { + + assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( + json -> json.node("id") + .withFailMessage("Negotiation id is null") + .isEqualTo(contractNegotiationRequestId), + json -> json.node("state") + .withFailMessage("ContractNegotiation is not in CONFIRMED state.") + .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), + json -> json.node("contractAgreement.id") + .withFailMessage("contractAgreement.id is null") + .isNotNull() + ); + }); + + // Obtain contract agreement ID + var contractAgreementId = fetchNegotiatedAgreement(contractNegotiationRequestId) + .get("contractAgreement").get("id").textValue(); + + //Initiate file transfer + var transferProcessId = + given() + .noContentType() + .pathParam(FILE_NAME_PARAM, PROVIDER_ASSET_NAME) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) + .queryParam(CONTRACT_ID_PARAM, contractAgreementId) + .when() + .post(FILE_TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // Verify TransferProcessId + assertThat(transferProcessId).isNotNull(); + + //Verify file transfer is completed and file contents + await().atMost(30, SECONDS).untilAsserted(() -> { + assertThatJson(fetchTransfer(transferProcessId).toString()).and( + json -> json.node("id") + .withFailMessage("TransferProcessId not matched") + .isEqualTo(transferProcessId), + json -> json.node("state") + .withFailMessage("TransferProcess is not in COMPLETED state") + .isEqualTo(TransferProcessStates.COMPLETED.code()) + ); + }); + + if (CHECK_FILE) { + var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); + var actualFileContent = fetchFileContent(copiedFilePath); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are null") + .isNotNull(); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are not same as the source file") + .isEqualTo(fileContent); + } + } + + private ObjectNode fetchTransfer(String transferProcessId) { + return + given() + .pathParam(TRANSFER_ID_PARAM, transferProcessId) + .when() + .get(TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Fetch negotiated contract agreement. + * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. + * @return Negotiation as {@link ObjectNode}. + */ + private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) { + return + given() + .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) + .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) + .when() + .get(CONTRACT_AGREEMENT_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Helper method to read file contents on the given {@link Path} + * @param filePath see {@link Path} + * @return Contents of file as a {@link String} or null if file does not exist. + */ + private String fetchFileContent(Path filePath) { + if (filePath.toFile().exists()) { + try { + return Files.readAllLines(filePath).get(0); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} From e35fafef2b81b3b64efe1271442843da09fb5f93 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 09:14:42 +0100 Subject: [PATCH 21/35] Update SeparateClassloaderSystemTest.java --- .../samples/SeparateClassloaderSystemTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java index ee251ceeeca..115e67d5e26 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -94,6 +94,7 @@ static void setUp() throws Exception { System.setProperty("web.http.port", "9191"); System.setProperty("edc.api.control.auth.apikey.value", "password"); + System.setProperty("ids.webhook.address", "http://localhost:9191"); var c = new CountDownLatch(1); Thread t = new Thread(() -> { @@ -120,13 +121,17 @@ static void setUp() throws Exception { t.start(); c.await(10, SECONDS); System.clearProperty("web.http.port"); + System.setProperty("ids.webhook.address", "http://localhost:8181"); + // Consumer connector host URI. RestAssured.baseURI = format("http://%s:%s", "localhost", 9191); } @Test - public void transferFile_success() throws IOException { + public void transferFile_success() throws Exception { + Thread.sleep(4000); + //Arrange var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); //Create a file with test data on provide file system. From 65001f2a147f56ece8c984f5bf2ab6cc7feaa591 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 09:43:38 +0100 Subject: [PATCH 22/35] Update SeparateClassloaderSystemTest.java --- .../SeparateClassloaderSystemTest.java | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java index ee251ceeeca..c9af544dcd7 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -44,6 +45,7 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.jar.JarInputStream; import java.util.stream.Collectors; import static io.restassured.RestAssured.given; @@ -94,39 +96,43 @@ static void setUp() throws Exception { System.setProperty("web.http.port", "9191"); System.setProperty("edc.api.control.auth.apikey.value", "password"); - var c = new CountDownLatch(1); - Thread t = new Thread(() -> + System.setProperty("ids.webhook.address", "http://localhost:9191"); + var latch = new CountDownLatch(1); + var otherConnector = new Thread(() -> { - ClassLoader parent = null; - parent = ClassLoader.getSystemClassLoader(); - URLClassLoader child = null; try { - File file = new File(rootProject + - "/consumer/build/libs/consumer.jar"); + var file = new File(rootProject + "/consumer/build/libs/consumer.jar"); assertThat(file).canRead(); + var jar = new JarInputStream(new FileInputStream(file)); + var manifest = jar.getManifest(); + var mainClassName = manifest.getMainAttributes().getValue("Main-Class"); - child = URLClassLoader.newInstance(new URL[]{file.toURL() - }, - parent); - Thread.currentThread().setContextClassLoader(child); - Class r = child.loadClass(BaseRuntime.class.getCanonicalName()); - var m = r.getMethod("main", String[].class); - m.invoke(null, new Object[]{new String[0]}); - c.countDown(); + var classLoader = URLClassLoader.newInstance(new URL[]{file.toURI().toURL()}, + ClassLoader.getSystemClassLoader()); + Thread.currentThread().setContextClassLoader(classLoader); + + var mainClass = classLoader.loadClass(mainClassName); + var mainMethod = mainClass.getMethod("main", String[].class); + mainMethod.invoke(null, new Object[]{new String[0]}); + + latch.countDown(); } catch (Exception e) { throw new RuntimeException(e); } }); - t.start(); - c.await(10, SECONDS); + otherConnector.start(); + latch.await(10, SECONDS); System.clearProperty("web.http.port"); + System.setProperty("ids.webhook.address", "http://localhost:8181"); // Consumer connector host URI. RestAssured.baseURI = format("http://%s:%s", "localhost", 9191); } @Test - public void transferFile_success() throws IOException { + public void transferFile_success() throws Exception { + Thread.sleep(4000); + //Arrange var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); //Create a file with test data on provide file system. @@ -225,6 +231,7 @@ private ObjectNode fetchTransfer(String transferProcessId) { /** * Fetch negotiated contract agreement. + * * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. * @return Negotiation as {@link ObjectNode}. */ @@ -242,6 +249,7 @@ private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) /** * Helper method to read file contents on the given {@link Path} + * * @param filePath see {@link Path} * @return Contents of file as a {@link String} or null if file does not exist. */ From 9d5d5cfd1e196616e8cb65982f25f09e1485f659 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 10:28:52 +0100 Subject: [PATCH 23/35] Isolate jar extension --- .../samples/EdcRuntimeExtension.java | 69 +++++++++++++++++++ .../SeparateClassloaderSystemTest.java | 68 ++++-------------- 2 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java new file mode 100644 index 00000000000..ce46bd88c0c --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java @@ -0,0 +1,69 @@ +package org.eclipse.dataspaceconnector.samples; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.io.File; +import java.io.FileInputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.jar.JarInputStream; +import java.util.stream.Collectors; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +public class EdcRuntimeExtension implements BeforeAllCallback, AfterAllCallback { + final Map properties; + private Thread otherConnector; + + public EdcRuntimeExtension(Map properties) { + this.properties = Map.copyOf(properties); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); + + var saved = (Properties) System.getProperties().clone(); + properties.forEach((k, v) -> System.setProperty(k, v)); + var latch = new CountDownLatch(1); + otherConnector = new Thread(() -> + { + try { + var file = new File(rootProject + "/consumer/build/libs/consumer.jar"); + assertThat(file).canRead(); + var jar = new JarInputStream(new FileInputStream(file)); + var manifest = jar.getManifest(); + var mainClassName = manifest.getMainAttributes().getValue("Main-Class"); + + var classLoader = URLClassLoader.newInstance(new URL[]{file.toURI().toURL()}, + ClassLoader.getSystemClassLoader()); + Thread.currentThread().setContextClassLoader(classLoader); + + var mainClass = classLoader.loadClass(mainClassName); + var mainMethod = mainClass.getMethod("main", String[].class); + mainMethod.invoke(null, new Object[]{new String[0]}); + + latch.countDown(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + otherConnector.start(); + latch.await(10, SECONDS); + System.setProperties(saved); + System.out.println("PROPS:" + System.getProperties()); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + + } +} diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java index c9af544dcd7..d8f10a97575 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -20,33 +20,18 @@ import org.apache.http.HttpStatus; import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; -import org.eclipse.dataspaceconnector.core.system.runtime.BaseRuntime; import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; -import org.eclipse.dataspaceconnector.spi.system.ServiceExtension; import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.RegisterExtension; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ServiceLoader; +import java.util.Map; import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.jar.JarInputStream; -import java.util.stream.Collectors; import static io.restassured.RestAssured.given; import static java.lang.String.format; @@ -61,7 +46,6 @@ */ @Tag("SystemTests") @IntegrationTest -@ExtendWith(EdcExtension.class) public class SeparateClassloaderSystemTest { private static final String PROVIDER_ASSET_NAME = "test-document"; @@ -78,7 +62,6 @@ public class SeparateClassloaderSystemTest { private static final String CONTRACT_ID_PARAM = "contractId"; private static final String FILE_NAME_PARAM = "filename"; - private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); @@ -88,41 +71,20 @@ public class SeparateClassloaderSystemTest { private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); private static final String API_KEY_HEADER = "X-Api-Key"; - @BeforeAll - static void setUp() throws Exception { - - // Prepare Testcontainer of consumer connector - var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); - - System.setProperty("web.http.port", "9191"); - System.setProperty("edc.api.control.auth.apikey.value", "password"); - System.setProperty("ids.webhook.address", "http://localhost:9191"); - var latch = new CountDownLatch(1); - var otherConnector = new Thread(() -> - { - try { - var file = new File(rootProject + "/consumer/build/libs/consumer.jar"); - assertThat(file).canRead(); - var jar = new JarInputStream(new FileInputStream(file)); - var manifest = jar.getManifest(); - var mainClassName = manifest.getMainAttributes().getValue("Main-Class"); + @RegisterExtension + @Order(1) + static EdcRuntimeExtension otherConnector = new EdcRuntimeExtension( + Map.of( + "web.http.port", "9191", + "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, + "ids.webhook.address", "http://localhost:9191")); - var classLoader = URLClassLoader.newInstance(new URL[]{file.toURI().toURL()}, - ClassLoader.getSystemClassLoader()); - Thread.currentThread().setContextClassLoader(classLoader); + @RegisterExtension + @Order(2) + static EdcExtension edc = new EdcExtension(); - var mainClass = classLoader.loadClass(mainClassName); - var mainMethod = mainClass.getMethod("main", String[].class); - mainMethod.invoke(null, new Object[]{new String[0]}); - - latch.countDown(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - otherConnector.start(); - latch.await(10, SECONDS); - System.clearProperty("web.http.port"); + @BeforeAll + static void setUp() { System.setProperty("ids.webhook.address", "http://localhost:8181"); // Consumer connector host URI. From d2f275991d5ac90010f80208a6972d50aeb611c3 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 10:46:17 +0100 Subject: [PATCH 24/35] Make JAR file configurable --- .../samples/EdcRuntimeExtension.java | 17 +++++++---------- .../samples/SeparateClassloaderSystemTest.java | 1 + 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java index ce46bd88c0c..dde19c5e833 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java @@ -13,31 +13,29 @@ import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.jar.JarInputStream; -import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; public class EdcRuntimeExtension implements BeforeAllCallback, AfterAllCallback { + final String jarFile; final Map properties; private Thread otherConnector; - public EdcRuntimeExtension(Map properties) { + public EdcRuntimeExtension(String jarFile, Map properties) { + this.jarFile = jarFile; this.properties = Map.copyOf(properties); } @Override public void beforeAll(ExtensionContext context) throws Exception { - var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); - - var saved = (Properties) System.getProperties().clone(); + var savedProperties = (Properties) System.getProperties().clone(); properties.forEach((k, v) -> System.setProperty(k, v)); var latch = new CountDownLatch(1); otherConnector = new Thread(() -> { try { - var file = new File(rootProject + "/consumer/build/libs/consumer.jar"); + var file = new File(jarFile); assertThat(file).canRead(); var jar = new JarInputStream(new FileInputStream(file)); var manifest = jar.getManifest(); @@ -58,12 +56,11 @@ public void beforeAll(ExtensionContext context) throws Exception { }); otherConnector.start(); latch.await(10, SECONDS); - System.setProperties(saved); - System.out.println("PROPS:" + System.getProperties()); + System.setProperties(savedProperties); } @Override public void afterAll(ExtensionContext context) throws Exception { - + otherConnector.join(); } } diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java index d8f10a97575..68b1a3e99f4 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -74,6 +74,7 @@ public class SeparateClassloaderSystemTest { @RegisterExtension @Order(1) static EdcRuntimeExtension otherConnector = new EdcRuntimeExtension( + "../consumer/build/libs/consumer.jar", Map.of( "web.http.port", "9191", "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, From 6d78b29d2facb989bda8a4cb70be9750207445a6 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 15:43:11 +0100 Subject: [PATCH 25/35] Alternative classpath based implementation --- build.gradle.kts | 7 ++ .../samples/GradleModuleRuntimeExtension.java | 104 ++++++++++++++++++ ...xtension.java => JarRuntimeExtension.java} | 19 ++-- .../SeparateClassloaderSystemTest.java | 14 ++- 4 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java rename samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/{EdcRuntimeExtension.java => JarRuntimeExtension.java} (83%) diff --git a/build.gradle.kts b/build.gradle.kts index b028d945c1e..1c5b10a7c2d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,6 +79,13 @@ allprojects { } } + + tasks.register("printClasspath") { + doLast { + println("${sourceSets["main"].runtimeClasspath.asPath}"); + } + } + pluginManager.withPlugin("java-library") { group = groupId version = edcVersion diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java new file mode 100644 index 00000000000..c9b74d2eac2 --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java @@ -0,0 +1,104 @@ +package org.eclipse.dataspaceconnector.samples; + +import org.eclipse.dataspaceconnector.core.system.runtime.BaseRuntime; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.io.File; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +public class GradleModuleRuntimeExtension implements BeforeAllCallback, AfterAllCallback { + final String moduleName; + final Map properties; + private Thread runtimeThread; + + public GradleModuleRuntimeExtension(String moduleName, Map properties) { + this.moduleName = moduleName; + this.properties = Map.copyOf(properties); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + + var root = new File("../../..").getCanonicalFile(); + Process exec = Runtime.getRuntime().exec(root + "/gradlew -q " + moduleName + ":printClasspath"); + InputStream inputStream = exec.getInputStream(); + var st = new String(inputStream.readAllBytes()); + assertThat(exec.exitValue()).isEqualTo(0); + + var classPathEntries = Arrays.stream(st.split(":|\\s")) + .filter(s -> !s.isBlank()) + .flatMap(p -> resolveClassPathEntry(root, p)) + .toArray(URL[]::new); + + var classLoader = URLClassLoader.newInstance(classPathEntries, + ClassLoader.getSystemClassLoader()); + + var mainClassName = BaseRuntime.class.getCanonicalName(); + var mainClass = classLoader.loadClass(mainClassName); + var mainMethod = mainClass.getMethod("main", String[].class); + + var savedProperties = (Properties) System.getProperties().clone(); + properties.forEach(System::setProperty); + var latch = new CountDownLatch(1); + runtimeThread = new Thread(() -> + { + try { + Thread.currentThread().setContextClassLoader(classLoader); + mainMethod.invoke(null, new Object[]{new String[0]}); + latch.countDown(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + runtimeThread.start(); + + assertThat(latch.await(10, SECONDS)).isTrue(); + + System.setProperties(savedProperties); + } + + private static Stream resolveClassPathEntry(File root, String classPathEntry) { + try { + File f = new File(classPathEntry).getCanonicalFile(); + + // If class path entry is not a JAR unter the root (a sub-project), do not transform it + boolean isUnderRoot = f.getCanonicalPath().startsWith(root.getCanonicalPath() + File.separator); + if (!classPathEntry.toLowerCase(Locale.ROOT).endsWith(".jar") || !f.isFile() || !isUnderRoot) { + return Stream.of(toURL(classPathEntry)); + } + + // Replace JAR entry with the resolved classes and resources folder + var buildDir = f.getParentFile().getParent(); + return Stream.of( + toURL(buildDir + "/classes/java/main/"), + toURL(buildDir + "/resources/main/") + ); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static URL toURL(String s) throws MalformedURLException { + return new URL("file:" + s); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + runtimeThread.join(); + } +} diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/JarRuntimeExtension.java similarity index 83% rename from samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java rename to samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/JarRuntimeExtension.java index dde19c5e833..715f698aa84 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/EdcRuntimeExtension.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/JarRuntimeExtension.java @@ -8,7 +8,6 @@ import java.io.FileInputStream; import java.net.URL; import java.net.URLClassLoader; -import java.nio.file.Paths; import java.util.Map; import java.util.Properties; import java.util.concurrent.CountDownLatch; @@ -17,12 +16,12 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; -public class EdcRuntimeExtension implements BeforeAllCallback, AfterAllCallback { +public class JarRuntimeExtension implements BeforeAllCallback, AfterAllCallback { final String jarFile; final Map properties; - private Thread otherConnector; + private Thread runtimeThread; - public EdcRuntimeExtension(String jarFile, Map properties) { + public JarRuntimeExtension(String jarFile, Map properties) { this.jarFile = jarFile; this.properties = Map.copyOf(properties); } @@ -30,9 +29,9 @@ public EdcRuntimeExtension(String jarFile, Map properties) { @Override public void beforeAll(ExtensionContext context) throws Exception { var savedProperties = (Properties) System.getProperties().clone(); - properties.forEach((k, v) -> System.setProperty(k, v)); + properties.forEach(System::setProperty); var latch = new CountDownLatch(1); - otherConnector = new Thread(() -> + runtimeThread = new Thread(() -> { try { var file = new File(jarFile); @@ -54,13 +53,15 @@ public void beforeAll(ExtensionContext context) throws Exception { throw new RuntimeException(e); } }); - otherConnector.start(); - latch.await(10, SECONDS); + runtimeThread.start(); + + assertThat(latch.await(10, SECONDS)).isTrue(); + System.setProperties(savedProperties); } @Override public void afterAll(ExtensionContext context) throws Exception { - otherConnector.join(); + runtimeThread.join(); } } diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java index 68b1a3e99f4..14654316ff4 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java @@ -73,12 +73,24 @@ public class SeparateClassloaderSystemTest { @RegisterExtension @Order(1) - static EdcRuntimeExtension otherConnector = new EdcRuntimeExtension( + static GradleModuleRuntimeExtension otherConnector = new GradleModuleRuntimeExtension( + ":samples:04.0-file-transfer:consumer", + Map.of( + "web.http.port", "9191", + "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, + "ids.webhook.address", "http://localhost:9191")); + + // Alternative to the above: + /* + @RegisterExtension + @Order(1) + static JarRuntimeExtension otherConnector = new JarRuntimeExtension( "../consumer/build/libs/consumer.jar", Map.of( "web.http.port", "9191", "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, "ids.webhook.address", "http://localhost:9191")); + */ @RegisterExtension @Order(2) From 68b0556db23714a7ac1e92f8aa08712d73cba098 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 15:49:37 +0100 Subject: [PATCH 26/35] Update GradleModuleRuntimeExtension.java --- .../samples/GradleModuleRuntimeExtension.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java index c9b74d2eac2..2c99d9987b6 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java @@ -37,7 +37,7 @@ public void beforeAll(ExtensionContext context) throws Exception { Process exec = Runtime.getRuntime().exec(root + "/gradlew -q " + moduleName + ":printClasspath"); InputStream inputStream = exec.getInputStream(); var st = new String(inputStream.readAllBytes()); - assertThat(exec.exitValue()).isEqualTo(0); + assertThat(exec.waitFor()).isEqualTo(0); var classPathEntries = Arrays.stream(st.split(":|\\s")) .filter(s -> !s.isBlank()) From bf48a69fc93ea87c434559bc407de422e1304440 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Thu, 3 Feb 2022 16:34:39 +0100 Subject: [PATCH 27/35] Add docker-compose approach for sample04 test --- samples/04.0-file-transfer/Dockerfile | 27 +++++++++++++++++++ samples/04.0-file-transfer/docker-compose.yml | 24 +++++++++++++++++ .../samples/FileTransferSystemTest.java | 9 ++++--- 3 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 samples/04.0-file-transfer/Dockerfile create mode 100644 samples/04.0-file-transfer/docker-compose.yml diff --git a/samples/04.0-file-transfer/Dockerfile b/samples/04.0-file-transfer/Dockerfile new file mode 100644 index 00000000000..9f1d0241119 --- /dev/null +++ b/samples/04.0-file-transfer/Dockerfile @@ -0,0 +1,27 @@ +FROM openjdk:11-jre-slim AS runtime + +WORKDIR /build + +COPY ./consumer/build/libs/consumer.jar /build/consumer/ +RUN echo "web.http.port=9191\n" \ + "edc.api.control.auth.apikey.value=password\n" \ + "ids.webhook.address=http://sample04-connector-consumer:9191\n" > /build/consumer/config.properties + +COPY ./provider/build/libs/provider.jar /build/provider/ +RUN echo "edc.samples.04.asset.path=/tmp/provider/test-document.txt\n" \ + "ids.webhook.address=http://sample04-connector-provider:8181\n" > /build/provider/config.properties + + +FROM runtime AS sample04-connector-provider + +WORKDIR /app +COPY --from=runtime /build/provider/* /app/ + +ENTRYPOINT ["java", "-Dedc.fs.config=config.properties", "-jar", "provider.jar"] + +FROM runtime AS sample04-connector-consumer + +WORKDIR /app +COPY --from=runtime /build/consumer/* /app/ + +ENTRYPOINT ["java", "-Dedc.fs.config=config.properties", "-jar", "consumer.jar"] \ No newline at end of file diff --git a/samples/04.0-file-transfer/docker-compose.yml b/samples/04.0-file-transfer/docker-compose.yml new file mode 100644 index 00000000000..8c1998f1960 --- /dev/null +++ b/samples/04.0-file-transfer/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.8' + +services: + sample04-connector-provider: + container_name: sample04-connector-provider + build: + context: . + target: sample04-connector-provider + ports: + - "8181:8181" + volumes: + - /tmp/provider:/tmp/provider + - /tmp/consumer:/tmp/consumer + + sample04-connector-consumer: + container_name: sample04-connector-consumer + build: + context: . + target: sample04-connector-consumer + ports: + - "9191:9191" + volumes: + - /tmp/provider:/tmp/provider + - /tmp/consumer:/tmp/consumer diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java index dc2770ea6e8..79b7c43485f 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferSystemTest.java @@ -62,10 +62,11 @@ public class FileTransferSystemTest { private static final String FILE_NAME_PARAM = "filename"; private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); - private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); + private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.consumer.asset.path", "/tmp/consumer"); + private static final String CONSUMER_ASSET_HOST_PATH = propOrEnv("edc.consumer.asset.host.path", "/tmp/consumer"); private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); - private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); + private static final String PROVIDER_ASSET_HOST_PATH = propOrEnv("edc.provider.asset.host.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); @@ -82,7 +83,7 @@ public void transferFile_success() throws IOException { var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); //Create a file with test data on provide file system. var fileContent = "Sample04-test-" + UUID.randomUUID(); - Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); + Files.write(Path.of(PROVIDER_ASSET_HOST_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); //Act & Assert @@ -152,7 +153,7 @@ public void transferFile_success() throws IOException { }); if (CHECK_FILE) { - var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); + var copiedFilePath = Path.of(format(CONSUMER_ASSET_HOST_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); var actualFileContent = fetchFileContent(copiedFilePath); assertThat(actualFileContent) .withFailMessage("Transferred file contents are null") From eaeea4243fa46cf843b0a97657205b61c3ed2e1e Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 17:41:04 +0100 Subject: [PATCH 28/35] Update GradleModuleRuntimeExtension.java --- .../samples/GradleModuleRuntimeExtension.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java index 2c99d9987b6..34fc70cfcbc 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/GradleModuleRuntimeExtension.java @@ -1,6 +1,6 @@ package org.eclipse.dataspaceconnector.samples; -import org.eclipse.dataspaceconnector.core.system.runtime.BaseRuntime; +import org.eclipse.dataspaceconnector.boot.system.runtime.BaseRuntime; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; From 0219dbffe39ff3298310b8ce9078cc7bfa721220 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 17:55:20 +0100 Subject: [PATCH 29/35] Create 2022-02-03-integration-testing.md --- .../2022-02-03-integration-testing.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/developer/decision-records/2022-02-03-integration-testing.md diff --git a/docs/developer/decision-records/2022-02-03-integration-testing.md b/docs/developer/decision-records/2022-02-03-integration-testing.md new file mode 100644 index 00000000000..a7eaf6cc57e --- /dev/null +++ b/docs/developer/decision-records/2022-02-03-integration-testing.md @@ -0,0 +1,49 @@ +# Integration testing + +## Decision + +Build a + +## Rationale + +The need to provide an integration test harness that supports multiple runtimes emerges from multiple needs: + +- Stabilizing [samples](../../samples) that run multiple connectors, which have been breaking frequently. +- Testing system behavior when multiple connectors interact, e.g. the contract negotiation process. +- Testing system behavior upon component failure. +- Providing a test facility for factoring out application components to separate runtimes (e.g. DPF). + +Key drivers for the choice are: + +- Fast and efficient run in CI. +- Fast inner loop and debuggability for developers. +- Use of existing frameworks, stability and portability. + +We have performed technical spikes testing multiple approaches (detailed further below), including: +- Docker +- Docker compose +- Testcontainers +- JUnit + +Spinning additional Class Loaders for runtimes provides very fast inner loop. Using the Gradle Classpath +is DRY and ensures the runtime under test exactly matches the standalone one. + +In contrast, approaches based on Docker have a slow inner loop and require rebuild between runs. +Support for bidirectional communication with Testcontainers is clunky and complex. + +The approach used is not limited to the Dataspace Connector, it can be used to run any Java module +if required in the future. + +## Spikes + +We have performed technical spikes: + +### Docker with testcontainers + +### Docker-compose with testcontainers + +### Docker-compose + +### Class Loader with Shadow JAR + +### Class Loader with Gradle Classpath From 67ee69362ee8798168d6d20c2c2cf6596feddeff Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 3 Feb 2022 20:00:31 +0100 Subject: [PATCH 30/35] ADR --- .../2022-02-03-integration-testing.md | 281 +++++++++++++++++- ...> ClassLoaderWithGradleClasspathTest.java} | 18 +- .../samples/ClassLoaderWithShadowJarTest.java | 227 ++++++++++++++ 3 files changed, 496 insertions(+), 30 deletions(-) rename samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/{SeparateClassloaderSystemTest.java => ClassLoaderWithGradleClasspathTest.java} (94%) create mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithShadowJarTest.java diff --git a/docs/developer/decision-records/2022-02-03-integration-testing.md b/docs/developer/decision-records/2022-02-03-integration-testing.md index a7eaf6cc57e..dfee6c0593a 100644 --- a/docs/developer/decision-records/2022-02-03-integration-testing.md +++ b/docs/developer/decision-records/2022-02-03-integration-testing.md @@ -2,7 +2,11 @@ ## Decision -Build a +Extend the existing `EdcExtension` JUnit facility (that can currently run a single EDC runtime and supports stubbing and extension of runtime services) as follows: + +- Load more than one EDC runtime. +- Load EDC runtimes in separate Class Loaders. +- Run each EDC runtime Class Loader with its effective runtime class path, based on its module's Gradle configuration. ## Rationale @@ -11,39 +15,288 @@ The need to provide an integration test harness that supports multiple runtimes - Stabilizing [samples](../../samples) that run multiple connectors, which have been breaking frequently. - Testing system behavior when multiple connectors interact, e.g. the contract negotiation process. - Testing system behavior upon component failure. -- Providing a test facility for factoring out application components to separate runtimes (e.g. DPF). +- Providing a test facility for factoring out application components to separate runtimes (e.g. [DPF](https://github.com/eclipse-dataspaceconnector/DataSpaceConnector/issues/463)). Key drivers for the choice are: - Fast and efficient run in CI. -- Fast inner loop and debuggability for developers. +- Fast "inner loop" (i.e. ability to quickly rerun tests after changing code) and debuggability for developers. - Use of existing frameworks, stability and portability. -We have performed technical spikes testing multiple approaches (detailed further below), including: -- Docker -- Docker compose -- Testcontainers +We have performed technical spikes testing multiple approaches (detailed further below), including various combinations of: - JUnit +- [Docker compose](https://docs.docker.com/compose/) +- Starting custom Class Loaders for separate threads for the Provider and Connector with separate class paths, and distinct instances of dependent instances, effectively providing full runtime isolation within a single JVM +- [Testcontainers](https://www.testcontainers.org/) with custom containers was also evaluated, but support for bidirectional communication to host is complex, and we didn't manage to get it running. -Spinning additional Class Loaders for runtimes provides very fast inner loop. Using the Gradle Classpath -is DRY and ensures the runtime under test exactly matches the standalone one. +Spinning additional Class Loaders for runtimes with JUnit provides very fast inner loop. Using the Gradle Classpath is [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) and ensures the runtime under test exactly matches the standalone one. In contrast, approaches based on Docker have a slow inner loop and require rebuild between runs. -Support for bidirectional communication with Testcontainers is clunky and complex. The approach used is not limited to the Dataspace Connector, it can be used to run any Java module if required in the future. ## Spikes -We have performed technical spikes: +We have performed technical spikes on [sample 04.0-file-transfer](../../../samples/04.0-file-transfer/README.md), that runs two EDC connectors, a Consumer and a Provider. The test requires three components: + +- Consumer EDC connector +- Provider EDC connector +- HTTP client code to interact with the Consumer provider API -### Docker with testcontainers +We have written code for the following three options: -### Docker-compose with testcontainers +- **Docker-compose,** which works but provides an inconvenient inner loop. +- **Class Loader with Gradle Classpath**, which uses Gradle to determine the effective class path for each runtime, and is very efficient. It is therefore the base for the recommendation above. +- **Class Loader with Shadow JAR**, a variant of the above which uses the Shadow JAR, and provides a less efficient inner loop. ### Docker-compose -### Class Loader with Shadow JAR +In this setup, we run both connectors in Docker containers. Once they are up, we run HTTP client code in a JUnit test running on the host. + +```yaml +# docker-compose.yaml +services: + sample04-connector-provider: + build: + context: . + target: sample04-connector-provider + ports: + - "8181:8181" + volumes: + - /tmp/provider:/tmp/provider + - /tmp/consumer:/tmp/consumer + + sample04-connector-consumer: + build: + context: . + target: sample04-connector-consumer + ports: + - "9191:9191" + volumes: + - /tmp/provider:/tmp/provider + - /tmp/consumer:/tmp/consumer + +``` + +```shell +$ (cd samples/04.0-file-transfer && docker-compose up -d) + +[...] +Successfully built 26b639e8f852 +Successfully tagged 040-file-transfer_sample04-connector-consumer:latest + +[+] Running 3/3 + ⠿ Network 040-file-transfer_default Created 0.0s + ⠿ Container sample04-connector-provider Started 0.7s + ⠿ Container sample04-connector-consumer Started 0.6s +``` + +```shell +$ RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest +``` + +This setup is stable and straightforward, but the inner loop is not very efficient. Debugging a remote process is possible buts adds complexity. ### Class Loader with Gradle Classpath + +The integration tests module only contains the classpath of the Provider module: + +```kotlin + implementation(project(":samples:04.0-file-transfer:provider")) +``` + +The JUnit integration test runs the Provider using the preexisting `EdcExtension`, and the Consumer using a newly developed extension: + +```java + // EDC Consumer runtime + @RegisterExtension + @Order(1) + static GradleModuleRuntimeExtension otherConnector = new GradleModuleRuntimeExtension( + ":samples:04.0-file-transfer:consumer", // Gradle module of the runtime to be started + Map.of( // settings + "web.http.port", "9191", + "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, + "ids.webhook.address", "http://localhost:9191")); + + // EDC Provider runtime + @RegisterExtension + @Order(2) + static EdcExtension edc = new EdcExtension(); + +``` + +The extension determines the class path by running a custom Gradle task: + +```java +// GradleModuleRuntimeExtension.java +Runtime.getRuntime().exec(root + "/gradlew -q " + moduleName + ":printClasspath"); + +// ... process classpath (see below) ... + +// run a thread with a custom class loader +var classLoader = URLClassLoader.newInstance(classPathEntries, + ClassLoader.getSystemClassLoader()); + +var mainClassName = BaseRuntime.class.getCanonicalName(); +var mainClass = classLoader.loadClass(mainClassName); +var mainMethod = mainClass.getMethod("main", String[].class); + +runtimeThread = new Thread(() -> + { + Thread.currentThread().setContextClassLoader(classLoader); + mainMethod.invoke(null, new Object[]{new String[0]}); + }) +``` + +```kotlin + // build.gradle.kts (under allprojects) + tasks.register("printClasspath") { + doLast { + println("${sourceSets["main"].runtimeClasspath.asPath}"); + } + } +``` + +The Gradle classpath is composed of JARs within the Connector project (e.g. `/path/to/EclipseDataSpaceConnector/extensions/api/control/build/libs/control-0.0.1-SNAPSHOT.jar`) and JARs from the Gradle cache (e.g. `/home/user/.gradle/caches/modules-2/files-2.1/org.eclipse.jetty/jetty-util/11.0.6/292fa5d7b2cef3483da8a7fa9dd608bfc9896564/jetty-util-11.0.6.jar`). The Extension code replaces JAR entries of Connector modules with their compiled classes and resource directories (`build/classes/java/main/` and `build/resources/main/`), and loads the connector within a separate Class Loader. + +Setting up the Test task to pass standard output: + +```kotlin +// samples/04.0-file-transfer/integration-tests/build.gradle.kts +tasks.getByName("test") { + testLogging { + showStandardStreams = true + } +} +``` + +The test runs in seconds. + +``` +$ RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.ClassLoaderWithGradleClasspathTest + +> Configure project : +> No version was specified, setting default 0.0.1-SNAPSHOT +> If you want to change this, supply the -Pversion=X.Y.Z parameter + + +> Task :samples:04.0-file-transfer:integration-tests:test + +ClassLoaderWithGradleClasspathTest STANDARD_OUT + INFO 2022-02-03T17:19:42.201218 Configuration file does not exist: dataspaceconnector-configuration.properties. Ignoring. + INFO 2022-02-03T17:19:42.21943 Initialized FS Configuration + INFO 2022-02-03T17:19:42.228993 Secrets vault not configured. Defaulting to null vault. + INFO 2022-02-03T17:19:42.229459 Initialized Null Vault + INFO 2022-02-03T17:19:42.469222 Initialized Core Services + INFO 2022-02-03T17:19:42.470778 Initialized In-Memory Asset Index + INFO 2022-02-03T17:19:42.472156 Initialized In-Memory Contract Definition Store + INFO 2022-02-03T17:19:42.480915 Initialized Core Contract Service + INFO 2022-02-03T17:19:42.481961 Initialized In-Memory Transfer Process Store + INFO 2022-02-03T17:19:42.483762 Initialized org.eclipse.dataspaceconnector.transfer.core.CommandExtension + INFO 2022-02-03T17:19:42.489996 Initialized Core Transfer + INFO 2022-02-03T17:19:42.490808 Initialized Mock IAM +[...] + INFO 2022-02-03T17:19:42.673829 HTTP listening on 9191 + INFO 2022-02-03T17:19:42.76275 Started Jetty Service + INFO 2022-02-03T17:19:43.228771 Registered Web API context at: /api/* + INFO 2022-02-03T17:19:43.229002 Started Jersey Web Service + INFO 2022-02-03T17:19:43.229442 Started IDS Multipart API + INFO 2022-02-03T17:19:43.22957 Started org.eclipse.dataspaceconnector.extensions.api.FileTransferExtension + INFO 2022-02-03T17:19:43.229659 Started IDS Multipart Dispatcher API + INFO 2022-02-03T17:19:43.229751 Started IDS Transform Extension + INFO 2022-02-03T17:19:43.229837 Started EDC Control API + INFO 2022-02-03T17:19:43.229915 Started API Endpoint + INFO 2022-02-03T17:19:43.230525 edc-ce5e1eb0-62ea-43d0-858a-e48118e97d85 ready + +ClassLoaderWithGradleClasspathTest > transferFile_success() STANDARD_OUT + INFO 2022-02-03T17:19:43.666544 Configuration file does not exist: dataspaceconnector-configuration.properties. Ignoring. + INFO 2022-02-03T17:19:43.666879 Initialized FS Configuration + INFO 2022-02-03T17:19:43.667783 Secrets vault not configured. Defaulting to null vault. + INFO 2022-02-03T17:19:43.667928 Initialized Null Vault + INFO 2022-02-03T17:19:43.677249 Initialized Core Services + INFO 2022-02-03T17:19:43.677423 Initialized In-Memory Asset Index + INFO 2022-02-03T17:19:43.677543 Initialized In-Memory Contract Definition Store + INFO 2022-02-03T17:19:43.677745 Initialized Core Contract Service + INFO 2022-02-03T17:19:43.677868 Initialized In-Memory Transfer Process Store + INFO 2022-02-03T17:19:43.677988 Initialized org.eclipse.dataspaceconnector.transfer.core.CommandExtension + INFO 2022-02-03T17:19:43.678098 Initialized Core Transfer + INFO 2022-02-03T17:19:43.678255 Initialized Mock IAM +[...] + INFO 2022-02-03T17:19:43.685601 Started In-Memory Contract Negotiation Store + INFO 2022-02-03T17:19:43.686003 HTTP listening on 8181 + INFO 2022-02-03T17:19:43.689505 Started Jetty Service + +ClassLoaderWithGradleClasspathTest > transferFile_success() STANDARD_OUT + INFO 2022-02-03T17:19:43.736455 Registered Web API context at: /api/* + INFO 2022-02-03T17:19:43.736607 Started Jersey Web Service + INFO 2022-02-03T17:19:43.736657 Started IDS Multipart API + INFO 2022-02-03T17:19:43.736695 Started org.eclipse.dataspaceconnector.extensions.api.FileTransferExtension + INFO 2022-02-03T17:19:43.736754 Started IDS Multipart Dispatcher API + INFO 2022-02-03T17:19:43.73684 Started IDS Transform Extension + DEBUG 2022-02-03T17:19:48.434665 [Consumer] ContractNegotiation initiated. aa9a36be-af41-4da4-b198-711389690558 is now in state INITIAL. + DEBUG 2022-02-03T17:19:52.85985 [Provider] ContractNegotiation initiated. f8b7bb63-cdb3-4c17-8969-aaa0b868d5b3 is now in state REQUESTED. + DEBUG 2022-02-03T17:19:52.864953 [Provider] Contract offer received. Will be approved. + DEBUG 2022-02-03T17:19:52.865395 [Provider] ContractNegotiation f8b7bb63-cdb3-4c17-8969-aaa0b868d5b3 is now in state CONFIRMING. + DEBUG 2022-02-03T17:19:52.876959 Response received from connector. Status 200 + DEBUG 2022-02-03T17:19:52.889059 [Consumer] ContractNegotiation aa9a36be-af41-4da4-b198-711389690558 is now in state REQUESTED. + DEBUG 2022-02-03T17:19:53.743579 [Consumer] Contract agreement received. Validation successful. + DEBUG 2022-02-03T17:19:53.744268 [Consumer] ContractNegotiation aa9a36be-af41-4da4-b198-711389690558 is now in state CONFIRMED. + DEBUG 2022-02-03T17:19:53.748143 Response received from connector. Status 200 + DEBUG 2022-02-03T17:19:53.754331 [Provider] ContractNegotiation f8b7bb63-cdb3-4c17-8969-aaa0b868d5b3 is now in state CONFIRMED. + INFO 2022-02-03T17:19:53.831271 Received request for file test-document against provider http://localhost:8181/api/ids/multipart + DEBUG 2022-02-03T17:19:57.647989 Response received from connector. Status 200 + INFO 2022-02-03T17:19:57.650468 Object received: org.eclipse.dataspaceconnector.ids.api.multipart.dispatcher.message.MultipartRequestInProcessResponse@4eeb3196 + INFO 2022-02-03T17:19:58.687893 Copying data from File to File + INFO 2022-02-03T17:19:58.690052 Successfully copied file to /tmp/consumer/test-document.txt + DEBUG 2022-02-03T17:20:02.640684 Process 60e34667-c51b-4a38-bb08-ddd4d705c234 is now COMPLETED +[...] + INFO 2022-02-03T17:20:02.68389 Connector shutdown complete + +BUILD SUCCESSFUL in 33s +240 actionable tasks: 2 executed, 238 up-to-date +1.89user 0.11system 0:34.37elapsed 5%CPU +``` + +Running in an IDE (IntelliJ IDEA), the debugger can be used in both Provider and Consumer code. When a class source is modified and the test is rerun, the change is immediately reflected. + +We could have run the equivalent setup inverting the Provider and Consumer in the two extensions we used. In this infrastructure, only one of the connectors can use the `EdcExtension` facilities to add or mock services. This exploratory spike code was built as a quick-and-dirty PoC. When adapting this spike code for merging, we should improve this setup, by extending the `EdcExtension` to provide both classpath isolation and test runtime configurability. + +Note that this setup requires the modules to have previously been built (`./gradlew build`). + +### Class Loader with Shadow JAR + +The setup is very similar to the above, but the Shadow JAR of the connector is run: + +```java + // EDC Consumer runtime + @RegisterExtension + @Order(1) + static JarRuntimeExtension otherConnector = new JarRuntimeExtension( + "../consumer/build/libs/consumer.jar", + Map.of( + "web.http.port", "9191", + "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, + "ids.webhook.address", "http://localhost:9191")); + + + // EDC Provider runtime + @RegisterExtension + @Order(2) + static EdcExtension edc = new EdcExtension(); +``` + + + +```java +// JarRuntimeExtension +var classLoader = URLClassLoader.newInstance(new URL[]{jarFile.toURI().toURL()}, + ClassLoader.getSystemClassLoader()); +Thread.currentThread().setContextClassLoader(classLoader); + +var mainClass = classLoader.loadClass(mainClassName); +var mainMethod = mainClass.getMethod("main", String[].class); +mainMethod.invoke(null, new Object[]{new String[0]}); +``` + +The setup is slightly simpler than the previous one, but the inner loop is less efficient: the Gradle `shadowJar` task must be rerun between test runs. diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithGradleClasspathTest.java similarity index 94% rename from samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java rename to samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithGradleClasspathTest.java index 14654316ff4..033610e88a8 100644 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/SeparateClassloaderSystemTest.java +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithGradleClasspathTest.java @@ -18,7 +18,6 @@ import io.restassured.RestAssured; import io.restassured.http.ContentType; import org.apache.http.HttpStatus; -import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; @@ -45,8 +44,8 @@ * System Test for Sample 04.0-file-transfer */ @Tag("SystemTests") -@IntegrationTest -public class SeparateClassloaderSystemTest { +// @IntegrationTest +public class ClassLoaderWithGradleClasspathTest { private static final String PROVIDER_ASSET_NAME = "test-document"; @@ -80,18 +79,6 @@ public class SeparateClassloaderSystemTest { "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, "ids.webhook.address", "http://localhost:9191")); - // Alternative to the above: - /* - @RegisterExtension - @Order(1) - static JarRuntimeExtension otherConnector = new JarRuntimeExtension( - "../consumer/build/libs/consumer.jar", - Map.of( - "web.http.port", "9191", - "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, - "ids.webhook.address", "http://localhost:9191")); - */ - @RegisterExtension @Order(2) static EdcExtension edc = new EdcExtension(); @@ -115,7 +102,6 @@ public void transferFile_success() throws Exception { Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); //Act & Assert - // Initiate a contract negotiation var contractNegotiationRequestId = given() diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithShadowJarTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithShadowJarTest.java new file mode 100644 index 00000000000..b94746be643 --- /dev/null +++ b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/ClassLoaderWithShadowJarTest.java @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.dataspaceconnector.samples; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; +import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; +import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; +import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; + +/** + * System Test for Sample 04.0-file-transfer + */ +@Tag("SystemTests") +// @IntegrationTest +public class ClassLoaderWithShadowJarTest { + + private static final String PROVIDER_ASSET_NAME = "test-document"; + + private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; + private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; + private static final String TRANSFER_PATH = "/api/transfer/{transferId}"; + private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; + + private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; + private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; + private static final String TRANSFER_ID_PARAM = "transferId"; + private static final String DESTINATION_PARAM = "destination"; + private static final String CONTRACT_ID_PARAM = "contractId"; + private static final String FILE_NAME_PARAM = "filename"; + + private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); + + private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); + private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); + + private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); + private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); + private static final String API_KEY_HEADER = "X-Api-Key"; + + @RegisterExtension + @Order(1) + static JarRuntimeExtension otherConnector = new JarRuntimeExtension( + "../consumer/build/libs/consumer.jar", + Map.of( + "web.http.port", "9191", + "edc.api.control.auth.apikey.value", API_KEY_CONTROL_AUTH, + "ids.webhook.address", "http://localhost:9191")); + + @RegisterExtension + @Order(2) + static EdcExtension edc = new EdcExtension(); + + @BeforeAll + static void setUp() { + System.setProperty("ids.webhook.address", "http://localhost:8181"); + + // Consumer connector host URI. + RestAssured.baseURI = format("http://%s:%s", "localhost", 9191); + } + + @Test + public void transferFile_success() throws Exception { + Thread.sleep(4000); + + //Arrange + var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); + //Create a file with test data on provide file system. + var fileContent = "Sample04-test-" + UUID.randomUUID(); + Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); + + //Act & Assert + // Initiate a contract negotiation + var contractNegotiationRequestId = + given() + .contentType(ContentType.JSON) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .body(contractOffer) + .when() + .post(CONTRACT_NEGOTIATION_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // UUID is returned to get the contract agreement negotiated between provider and consumer. + assertThat(contractNegotiationRequestId) + .withFailMessage("Contract negotiation requestId is null").isNotBlank(); + + // Verify ContractNegotiation is CONFIRMED (state = 1200) + await().atMost(30, SECONDS).untilAsserted(() -> { + + assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( + json -> json.node("id") + .withFailMessage("Negotiation id is null") + .isEqualTo(contractNegotiationRequestId), + json -> json.node("state") + .withFailMessage("ContractNegotiation is not in CONFIRMED state.") + .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), + json -> json.node("contractAgreement.id") + .withFailMessage("contractAgreement.id is null") + .isNotNull() + ); + }); + + // Obtain contract agreement ID + var contractAgreementId = fetchNegotiatedAgreement(contractNegotiationRequestId) + .get("contractAgreement").get("id").textValue(); + + //Initiate file transfer + var transferProcessId = + given() + .noContentType() + .pathParam(FILE_NAME_PARAM, PROVIDER_ASSET_NAME) + .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) + .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) + .queryParam(CONTRACT_ID_PARAM, contractAgreementId) + .when() + .post(FILE_TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().asString(); + + // Verify TransferProcessId + assertThat(transferProcessId).isNotNull(); + + //Verify file transfer is completed and file contents + await().atMost(30, SECONDS).untilAsserted(() -> { + assertThatJson(fetchTransfer(transferProcessId).toString()).and( + json -> json.node("id") + .withFailMessage("TransferProcessId not matched") + .isEqualTo(transferProcessId), + json -> json.node("state") + .withFailMessage("TransferProcess is not in COMPLETED state") + .isEqualTo(TransferProcessStates.COMPLETED.code()) + ); + }); + + if (CHECK_FILE) { + var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); + var actualFileContent = fetchFileContent(copiedFilePath); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are null") + .isNotNull(); + assertThat(actualFileContent) + .withFailMessage("Transferred file contents are not same as the source file") + .isEqualTo(fileContent); + } + } + + private ObjectNode fetchTransfer(String transferProcessId) { + return + given() + .pathParam(TRANSFER_ID_PARAM, transferProcessId) + .when() + .get(TRANSFER_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Fetch negotiated contract agreement. + * + * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. + * @return Negotiation as {@link ObjectNode}. + */ + private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) { + return + given() + .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) + .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) + .when() + .get(CONTRACT_AGREEMENT_PATH) + .then() + .assertThat().statusCode(HttpStatus.SC_OK) + .extract().as(ObjectNode.class); + } + + /** + * Helper method to read file contents on the given {@link Path} + * + * @param filePath see {@link Path} + * @return Contents of file as a {@link String} or null if file does not exist. + */ + private String fetchFileContent(Path filePath) { + if (filePath.toFile().exists()) { + try { + return Files.readAllLines(filePath).get(0); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} From 8c6e42c4200bcd8081437400b386b8c420f9866b Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Fri, 4 Feb 2022 08:13:34 +0100 Subject: [PATCH 31/35] linting --- .../2022-02-03-integration-testing.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/developer/decision-records/2022-02-03-integration-testing.md b/docs/developer/decision-records/2022-02-03-integration-testing.md index dfee6c0593a..6d8021bfd05 100644 --- a/docs/developer/decision-records/2022-02-03-integration-testing.md +++ b/docs/developer/decision-records/2022-02-03-integration-testing.md @@ -2,7 +2,7 @@ ## Decision -Extend the existing `EdcExtension` JUnit facility (that can currently run a single EDC runtime and supports stubbing and extension of runtime services) as follows: +Extend the existing `EdcExtension` JUnit facility (that can currently run a single EDC runtime and supports stubbing and extension of runtime services) as follows: - Load more than one EDC runtime. - Load EDC runtimes in separate Class Loaders. @@ -24,6 +24,7 @@ Key drivers for the choice are: - Use of existing frameworks, stability and portability. We have performed technical spikes testing multiple approaches (detailed further below), including various combinations of: + - JUnit - [Docker compose](https://docs.docker.com/compose/) - Starting custom Class Loaders for separate threads for the Provider and Connector with separate class paths, and distinct instances of dependent instances, effectively providing full runtime isolation within a single JVM @@ -93,7 +94,7 @@ Successfully tagged 040-file-transfer_sample04-connector-consumer:latest ``` ```shell -$ RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest +RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest ``` This setup is stable and straightforward, but the inner loop is not very efficient. Debugging a remote process is possible buts adds complexity. @@ -106,7 +107,7 @@ The integration tests module only contains the classpath of the Provider module: implementation(project(":samples:04.0-file-transfer:provider")) ``` -The JUnit integration test runs the Provider using the preexisting `EdcExtension`, and the Consumer using a newly developed extension: +The JUnit integration test runs the Provider using the preexisting `EdcExtension`, and the Consumer using a newly developed extension: ```java // EDC Consumer runtime @@ -150,8 +151,8 @@ runtimeThread = new Thread(() -> ``` ```kotlin - // build.gradle.kts (under allprojects) - tasks.register("printClasspath") { + // build.gradle.kts (under allprojects) + tasks.register("printClasspath") { doLast { println("${sourceSets["main"].runtimeClasspath.asPath}"); } @@ -173,7 +174,7 @@ tasks.getByName("test") { The test runs in seconds. -``` +```shell $ RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.ClassLoaderWithGradleClasspathTest > Configure project : @@ -260,7 +261,7 @@ BUILD SUCCESSFUL in 33s Running in an IDE (IntelliJ IDEA), the debugger can be used in both Provider and Consumer code. When a class source is modified and the test is rerun, the change is immediately reflected. -We could have run the equivalent setup inverting the Provider and Consumer in the two extensions we used. In this infrastructure, only one of the connectors can use the `EdcExtension` facilities to add or mock services. This exploratory spike code was built as a quick-and-dirty PoC. When adapting this spike code for merging, we should improve this setup, by extending the `EdcExtension` to provide both classpath isolation and test runtime configurability. +We could have run the equivalent setup inverting the Provider and Consumer in the two extensions we used. In this infrastructure, only one of the connectors can use the `EdcExtension` facilities to add or mock services. This exploratory spike code was built as a quick-and-dirty PoC. When adapting this spike code for merging, we should improve this setup, by extending the `EdcExtension` to provide both classpath isolation and test runtime configurability. Note that this setup requires the modules to have previously been built (`./gradlew build`). @@ -286,8 +287,6 @@ The setup is very similar to the above, but the Shadow JAR of the connector is r static EdcExtension edc = new EdcExtension(); ``` - - ```java // JarRuntimeExtension var classLoader = URLClassLoader.newInstance(new URL[]{jarFile.toURI().toURL()}, From 9f7ef015c2e402dab23e89af26a849dbfa7e4e8f Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 4 Feb 2022 08:53:25 +0100 Subject: [PATCH 32/35] Update Sample04 Integration Test ADR --- .../decision-records/2022-02-03-integration-testing.md | 2 +- samples/04.0-file-transfer/integration-tests/build.gradle.kts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/developer/decision-records/2022-02-03-integration-testing.md b/docs/developer/decision-records/2022-02-03-integration-testing.md index 6d8021bfd05..77c5a0cc681 100644 --- a/docs/developer/decision-records/2022-02-03-integration-testing.md +++ b/docs/developer/decision-records/2022-02-03-integration-testing.md @@ -94,7 +94,7 @@ Successfully tagged 040-file-transfer_sample04-connector-consumer:latest ``` ```shell -RUN_INTEGRATION_TEST=true time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest +RUN_INTEGRATION_TEST=true EDC_PROVIDER_CONNECTOR_HOST=http://sample04-connector-provider:8181 time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest ``` This setup is stable and straightforward, but the inner loop is not very efficient. Debugging a remote process is possible buts adds complexity. diff --git a/samples/04.0-file-transfer/integration-tests/build.gradle.kts b/samples/04.0-file-transfer/integration-tests/build.gradle.kts index 8a0ccf1359d..54a872f3641 100644 --- a/samples/04.0-file-transfer/integration-tests/build.gradle.kts +++ b/samples/04.0-file-transfer/integration-tests/build.gradle.kts @@ -27,7 +27,6 @@ dependencies { testImplementation("org.assertj:assertj-core:3.22.0") testImplementation("org.awaitility:awaitility:4.1.1") testImplementation("net.javacrumbs.json-unit:json-unit-assertj:2.28.0") - testImplementation("org.testcontainers:testcontainers:1.16.3") testImplementation(testFixtures(project(":common:util"))) testImplementation(testFixtures(project(":launchers:junit"))) From 2665b6104f3f4fa6140e2902f5c02ee919d6cb09 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 4 Feb 2022 08:55:08 +0100 Subject: [PATCH 33/35] Remove Testcontainers based test --- ...leTransferWithTestContainerSystemTest.java | 240 ------------------ 1 file changed, 240 deletions(-) delete mode 100644 samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java diff --git a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java b/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java deleted file mode 100644 index 3e428e2d740..00000000000 --- a/samples/04.0-file-transfer/integration-tests/src/test/java/org/eclipse/dataspaceconnector/samples/FileTransferWithTestContainerSystemTest.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.dataspaceconnector.samples; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.restassured.RestAssured; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.dataspaceconnector.common.annotations.IntegrationTest; -import org.eclipse.dataspaceconnector.common.testfixtures.TestUtils; -import org.eclipse.dataspaceconnector.junit.launcher.EdcExtension; -import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates; -import org.eclipse.dataspaceconnector.spi.types.domain.transfer.TransferProcessStates; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.Testcontainers; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.images.builder.ImageFromDockerfile; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.UUID; - -import static io.restassured.RestAssured.given; -import static java.lang.String.format; -import static java.util.concurrent.TimeUnit.SECONDS; -import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.dataspaceconnector.common.configuration.ConfigurationFunctions.propOrEnv; - -/** - * System Test for Sample 04.0-file-transfer - */ -@Tag("SystemTests") -@IntegrationTest -@ExtendWith(EdcExtension.class) -public class FileTransferWithTestContainerSystemTest { - - private static final String PROVIDER_ASSET_NAME = "test-document"; - - private static final String CONTRACT_NEGOTIATION_PATH = "/api/negotiation"; - private static final String CONTRACT_AGREEMENT_PATH = "/api/control/negotiation/{contractNegotiationRequestId}"; - private static final String TRANSFER_PATH = "/api/transfer/{transferId}"; - private static final String FILE_TRANSFER_PATH = "/api/file/{filename}"; - - private static final String CONNECTOR_ADDRESS_PARAM = "connectorAddress"; - private static final String CONTRACT_NEGOTIATION_REQUEST_ID_PARAM = "contractNegotiationRequestId"; - private static final String TRANSFER_ID_PARAM = "transferId"; - private static final String DESTINATION_PARAM = "destination"; - private static final String CONTRACT_ID_PARAM = "contractId"; - private static final String FILE_NAME_PARAM = "filename"; - - private static final String CONSUMER_CONNECTOR_HOST = propOrEnv("edc.consumer.connector.host", "http://localhost:9191"); - private static final String CONSUMER_ASSET_PATH = propOrEnv("edc.samples.04.consumer.asset.path", "/tmp/consumer"); - - private static final String PROVIDER_CONNECTOR_HOST = propOrEnv("edc.provider.connector.host", "http://localhost:8181"); - private static final String PROVIDER_ASSET_PATH = propOrEnv("edc.samples.04.asset.path", format("/tmp/provider/%s.txt", PROVIDER_ASSET_NAME)); - - private static final String API_KEY_CONTROL_AUTH = propOrEnv("edc.api.control.auth.apikey.value", "password"); - private static final boolean CHECK_FILE = Boolean.parseBoolean(propOrEnv("CHECK_FILE", "true")); - private static final String API_KEY_HEADER = "X-Api-Key"; - private static GenericContainer consumerContainer; - - @BeforeAll - static void setUp() { - - // Enable Testcontainers to connect to host port - Testcontainers.exposeHostPorts(8181); - - // Prepare Testcontainer of consumer connector - var rootProject = Paths.get(System.getProperty("user.dir")).getParent().toAbsolutePath(); - consumerContainer = new GenericContainer( - new ImageFromDockerfile() - .withFileFromClasspath("Dockerfile", "ConsumerDockerfile") - .withFileFromPath("consumer.jar", Path.of(rootProject + "/consumer/build/libs/consumer.jar")) - .withFileFromPath("config.properties", Path.of(rootProject + "/consumer/config.properties"))) - .withExposedPorts(9191) - .withAccessToHost(true); - - // Start consumer connector test container - consumerContainer.start(); - // Consumer connector host URI. - RestAssured.baseURI = format("http://%s:%s", consumerContainer.getHost(), consumerContainer.getFirstMappedPort()); - } - - @AfterAll - static void tearDown() { - consumerContainer.stop(); - } - - @Test - public void transferFile_success() throws IOException { - //Arrange - var contractOffer = TestUtils.getFileFromResourceName("contractoffer.json"); - //Create a file with test data on provide file system. - var fileContent = "Sample04-test-" + UUID.randomUUID(); - Files.write(Path.of(PROVIDER_ASSET_PATH), fileContent.getBytes(StandardCharsets.UTF_8)); - - //Act & Assert - - // Initiate a contract negotiation - var contractNegotiationRequestId = - given() - .contentType(ContentType.JSON) - .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) - .body(contractOffer) - .when() - .post(CONTRACT_NEGOTIATION_PATH) - .then() - .assertThat().statusCode(HttpStatus.SC_OK) - .extract().asString(); - - // UUID is returned to get the contract agreement negotiated between provider and consumer. - assertThat(contractNegotiationRequestId) - .withFailMessage("Contract negotiation requestId is null").isNotBlank(); - - // Verify ContractNegotiation is CONFIRMED (state = 1200) - await().atMost(30, SECONDS).untilAsserted(() -> { - - assertThatJson(fetchNegotiatedAgreement(contractNegotiationRequestId).toString()).and( - json -> json.node("id") - .withFailMessage("Negotiation id is null") - .isEqualTo(contractNegotiationRequestId), - json -> json.node("state") - .withFailMessage("ContractNegotiation is not in CONFIRMED state.") - .isEqualTo(ContractNegotiationStates.CONFIRMED.code()), - json -> json.node("contractAgreement.id") - .withFailMessage("contractAgreement.id is null") - .isNotNull() - ); - }); - - // Obtain contract agreement ID - var contractAgreementId = fetchNegotiatedAgreement(contractNegotiationRequestId) - .get("contractAgreement").get("id").textValue(); - - //Initiate file transfer - var transferProcessId = - given() - .noContentType() - .pathParam(FILE_NAME_PARAM, PROVIDER_ASSET_NAME) - .queryParam(CONNECTOR_ADDRESS_PARAM, format("%s/api/ids/multipart", PROVIDER_CONNECTOR_HOST)) - .queryParam(DESTINATION_PARAM, CONSUMER_ASSET_PATH) - .queryParam(CONTRACT_ID_PARAM, contractAgreementId) - .when() - .post(FILE_TRANSFER_PATH) - .then() - .assertThat().statusCode(HttpStatus.SC_OK) - .extract().asString(); - - // Verify TransferProcessId - assertThat(transferProcessId).isNotNull(); - - //Verify file transfer is completed and file contents - await().atMost(30, SECONDS).untilAsserted(() -> { - assertThatJson(fetchTransfer(transferProcessId).toString()).and( - json -> json.node("id") - .withFailMessage("TransferProcessId not matched") - .isEqualTo(transferProcessId), - json -> json.node("state") - .withFailMessage("TransferProcess is not in COMPLETED state") - .isEqualTo(TransferProcessStates.COMPLETED.code()) - ); - }); - - if (CHECK_FILE) { - var copiedFilePath = Path.of(format(CONSUMER_ASSET_PATH + "/%s.txt", PROVIDER_ASSET_NAME)); - var actualFileContent = fetchFileContent(copiedFilePath); - assertThat(actualFileContent) - .withFailMessage("Transferred file contents are null") - .isNotNull(); - assertThat(actualFileContent) - .withFailMessage("Transferred file contents are not same as the source file") - .isEqualTo(fileContent); - } - } - - private ObjectNode fetchTransfer(String transferProcessId) { - return - given() - .pathParam(TRANSFER_ID_PARAM, transferProcessId) - .when() - .get(TRANSFER_PATH) - .then() - .assertThat().statusCode(HttpStatus.SC_OK) - .extract().as(ObjectNode.class); - } - - /** - * Fetch negotiated contract agreement. - * @param contractNegotiationRequestId ID of the ongoing contract negotiation between consumer and provider. - * @return Negotiation as {@link ObjectNode}. - */ - private ObjectNode fetchNegotiatedAgreement(String contractNegotiationRequestId) { - return - given() - .pathParam(CONTRACT_NEGOTIATION_REQUEST_ID_PARAM, contractNegotiationRequestId) - .header(API_KEY_HEADER, API_KEY_CONTROL_AUTH) - .when() - .get(CONTRACT_AGREEMENT_PATH) - .then() - .assertThat().statusCode(HttpStatus.SC_OK) - .extract().as(ObjectNode.class); - } - - /** - * Helper method to read file contents on the given {@link Path} - * @param filePath see {@link Path} - * @return Contents of file as a {@link String} or null if file does not exist. - */ - private String fetchFileContent(Path filePath) { - if (filePath.toFile().exists()) { - try { - return Files.readAllLines(filePath).get(0); - } catch (IOException e) { - e.printStackTrace(); - } - } - return null; - } -} From 409cfd8dba2605066e3e7f3cf289865f29c63043 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 4 Feb 2022 08:56:20 +0100 Subject: [PATCH 34/35] Delete old Dockerfile --- .../src/test/resources/ConsumerDockerfile | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile diff --git a/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile b/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile deleted file mode 100644 index c0432e2ac74..00000000000 --- a/samples/04.0-file-transfer/integration-tests/src/test/resources/ConsumerDockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM openjdk:11-jre-slim - -WORKDIR /build - -COPY ./consumer.jar . -COPY ./config.properties . - -EXPOSE 9191 - -ENTRYPOINT ["java", "-Dedc.fs.config=config.properties", "-jar", "consumer.jar"] \ No newline at end of file From 48f4ca82e1556995aab012f114057e8cac0f4551 Mon Sep 17 00:00:00 2001 From: Peeyush Chandel <555114+cpeeyush@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:07:47 +0100 Subject: [PATCH 35/35] Update Integration Testing ADR --- .../decision-records/2022-02-03-integration-testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/decision-records/2022-02-03-integration-testing.md b/docs/developer/decision-records/2022-02-03-integration-testing.md index 77c5a0cc681..246113138a8 100644 --- a/docs/developer/decision-records/2022-02-03-integration-testing.md +++ b/docs/developer/decision-records/2022-02-03-integration-testing.md @@ -97,7 +97,7 @@ Successfully tagged 040-file-transfer_sample04-connector-consumer:latest RUN_INTEGRATION_TEST=true EDC_PROVIDER_CONNECTOR_HOST=http://sample04-connector-provider:8181 time ./gradlew cleanTest :samples:04.0-file-transfer:integration-tests:test --tests org.eclipse.dataspaceconnector.samples.FileTransferSystemTest ``` -This setup is stable and straightforward, but the inner loop is not very efficient. Debugging a remote process is possible buts adds complexity. +This setup is stable and straightforward, but the inner loop is not very efficient. Debugging a remote process is possible buts adds complexity. For faster debugging one approach could be is to run connectors in debug mode within IDE and later use the same code to update docker build. ### Class Loader with Gradle Classpath