feat: bump edc version to 0.11.0-Snapshot (#96)

* feat: adapt connector testruntime for OTC E2E-Test #69 * feat: adapt OBS E2E-Test #69 * fix: add Annotations for Postgresql-Persistence #69 * feat: bump Version to 0.11.0-Snapshot #69 * bump: update dependencies Versions of external libs #69 * fix: adapt tests to version change #69 * chore: remove duplicated workflow #69 * chore: disable system-test for secret reorganization #69 * feat: resolve conflict between HTTP-Pull and OBS DataPlane #69
eclipse-edc · Nov 15, 2024 · c5e70b5 · c5e70b5
1 parent d4c3ec3
commit c5e70b5
Show file tree

Hide file tree

Showing 15 changed files with 140 additions and 173 deletions.
diff --git a/.github/workflows/gaussdb-tests.yaml b/.github/workflows/gaussdb-tests.yaml
@@ -1,3 +1,5 @@
+#Not Active in Use
+
 ---
 name: "Run GaussDB Tests"
 on:

diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -8,7 +8,7 @@ on:
 
 jobs:
 
-  secret-presence:
+  Secret-Presence:
     runs-on: ubuntu-latest
     outputs:
       HAS_OTC_CRED: ${{ steps.secret-presence.outputs.HAS_OTC_CRED }}
@@ -19,10 +19,11 @@ jobs:
         run: |
           [ ! -z "${{ secrets.obs_access_key_id }}" ] && 
           [ ! -z "${{ secrets.obs_secret_key }}" ] && echo "HAS_OTC_CRED=true" >> $GITHUB_OUTPUT
-          [ ! -z "${{ secrets.pg_connection_string }}" ] && echo "HAS_PG_CONNECTION_STRING=true" >> $GITHUB_OUTPUT
+          [ ! -z "${{ secrets.pg_connection_string }}" ] && echo "HAS_PG_CONNECTION_STRING=true" >> $GITHUB_OUTPUT &&
+          echo 
           exit 0
 
-  end-to-end-test:
+  End-to-End-Test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -31,19 +32,19 @@ jobs:
         run: |
           ./gradlew compileJava compileTestJava
           ./gradlew test -DincludeTags="EndToEndTest" -PverboseTest=true
-  obs-otc-test:
-    runs-on: ubuntu-latest
-    needs: [ secret-presence ]
-    if: |
-      needs.secret-presence.outputs.HAS_OTC_CRED
-    env:
-      OBS_ACCESS_KEY_ID: ${{ secrets.obs_access_key_id }}
-      OBS_SECRET_ACCESS_KEY: ${{ secrets.obs_secret_key }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-java
-      - name: OTC Cloud Tests
-        run: |
-          ./gradlew compileJava compileTestJava
-          ./gradlew test -DincludeTags="OtcTest" -PverboseTest=true
 
+#  Obs-Otc-Test:
+#    runs-on: ubuntu-latest
+#    needs: [ secret-presence ]
+#    if: |
+#      needs.secret-presence.outputs.HAS_OTC_CRED
+#    env:
+#      OBS_ACCESS_KEY_ID: ${{ secrets.OBS_ACCESS_KEY_ID }}
+#      OBS_SECRET_ACCESS_KEY: ${{ secrets.OBS_SECRET_ACCESS_KEY }}
+#    steps:
+#      - uses: actions/checkout@v4
+#      - uses: ./.github/actions/setup-java
+#      - name: OTC Cloud Tests
+#        run: |
+#          ./gradlew compileJava compileTestJava
+#          ./gradlew test -DincludeTags="OtcTest" -PverboseTest=true
diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml
diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
@@ -49,14 +49,9 @@ jobs:
       - name: Component Tests
         run: ./gradlew -DincludeTags="ComponentTest"
 
-  End-To-End-Tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: eclipse-edc/.github/.github/actions/setup-build@main
-
-      - name: End to End Integration Tests
-        run: ./gradlew test -DincludeTags="EndToEndTest"
+  Integration-Tests:
+    uses: ./.github/workflows/integration-tests.yaml
+    secrets: inherit
 
   API-Tests:
     runs-on: ubuntu-latest

diff --git a/e2e-tests/build.gradle.kts b/e2e-tests/build.gradle.kts
@@ -18,6 +18,8 @@ plugins {
 
 dependencies {
     testImplementation(libs.edc.junit)
+    testImplementation(libs.edc.lib.transform)
+    testImplementation(libs.edc.transform.dpf.signaling)
     testImplementation(testFixtures(libs.edc.testfixtures.managementapi))
     testImplementation(libs.restAssured)
     testImplementation(libs.awaitility)

diff --git a/e2e-tests/src/test/java/com/huawei/cloud/fixtures/HuaweiParticipant.java b/e2e-tests/src/test/java/com/huawei/cloud/fixtures/HuaweiParticipant.java
@@ -15,17 +15,13 @@
 package com.huawei.cloud.fixtures;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
-import jakarta.json.JsonArray;
 import org.eclipse.edc.connector.controlplane.test.system.utils.Participant;
 
 import java.net.URI;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 
-import static io.restassured.http.ContentType.JSON;
-import static org.awaitility.Awaitility.await;
 import static org.eclipse.edc.boot.BootServicesExtension.PARTICIPANT_ID;
 import static org.eclipse.edc.util.io.Ports.getFreePort;
 
@@ -59,6 +55,7 @@ public Map<String, String> controlPlaneConfiguration() {
                 put("web.http.public.port", String.valueOf(getFreePort()));
                 put("edc.dsp.callback.address", protocolEndpoint.getUrl().toString());
                 put("edc.connector.name", name);
+                put("edc.component.id", "connector-test");
                 put("edc.dataplane.token.validation.endpoint", "http://token-validation.com");
                 put("edc.dpf.selector.url", "http://does-this-matter.com");
                 put("edc.huawei.iam.endpoint", IAM_OTC_CLOUD_URL);
@@ -68,25 +65,6 @@ public Map<String, String> controlPlaneConfiguration() {
         };
     }
 
-    public JsonArray getPolicies() {
-
-        AtomicReference<JsonArray> array = new AtomicReference<>();
-        await().atMost(TIMEOUT).untilAsserted(() -> {
-            var response = managementEndpoint.baseRequest()
-                    .contentType(JSON)
-                    .when()
-                    .post("/v2/policydefinitions/request")
-                    .then()
-                    .log().ifError()
-                    .statusCode(200)
-                    .extract().body().asString();
-
-            array.set(objectMapper.readValue(response, JsonArray.class));
-        });
-
-        return array.get();
-    }
-
     public static final class Builder extends Participant.Builder<HuaweiParticipant, Builder> {
 
         private Builder() {

diff --git a/e2e-tests/src/test/java/com/huawei/cloud/tests/ObsTransferEndToEndTest.java b/e2e-tests/src/test/java/com/huawei/cloud/tests/ObsTransferEndToEndTest.java
@@ -14,17 +14,30 @@
 
 package com.huawei.cloud.tests;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.huawei.cloud.fixtures.HuaweiParticipant;
 import com.huawei.cloud.obs.ObsBucketSchema;
 import com.huawei.cloud.obs.TestFunctions;
 import com.obs.services.ObsClient;
 import io.restassured.http.ContentType;
+import jakarta.json.Json;
+import jakarta.json.JsonObject;
+import org.eclipse.edc.connector.api.signaling.transform.from.JsonObjectFromDataFlowStartMessageTransformer;
+import org.eclipse.edc.connector.api.signaling.transform.to.JsonObjectToDataFlowResponseMessageTransformer;
+import org.eclipse.edc.jsonld.util.JacksonJsonLd;
 import org.eclipse.edc.junit.annotations.EndToEndTest;
 import org.eclipse.edc.junit.extensions.EdcClassRuntimesExtension;
 import org.eclipse.edc.junit.extensions.EdcRuntimeExtension;
+import org.eclipse.edc.spi.result.Failure;
 import org.eclipse.edc.spi.types.domain.DataAddress;
 import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage;
 import org.eclipse.edc.spi.types.domain.transfer.FlowType;
+import org.eclipse.edc.spi.types.domain.transfer.TransferType;
+import org.eclipse.edc.transform.TypeTransformerRegistryImpl;
+import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
+import org.eclipse.edc.transform.transformer.dspace.from.JsonObjectFromDataAddressDspaceTransformer;
+import org.eclipse.edc.transform.transformer.dspace.to.JsonObjectToDataAddressDspaceTransformer;
+import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -33,9 +46,12 @@
 import org.testcontainers.junit.jupiter.Testcontainers;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.time.Duration;
+import java.util.Map;
 import java.util.UUID;
+import java.util.function.Function;
 
 import static io.restassured.RestAssured.given;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -62,6 +78,7 @@ public class ObsTransferEndToEndTest {
     private static final String CONSUMER_SK = "consumer-sk";
     private static final String PROVIDER_AK = "provider-ak";
     private static final String PROVIDER_SK = "provider-sk";
+
     @RegisterExtension
     static EdcClassRuntimesExtension runtimes = new EdcClassRuntimesExtension(
             new EdcRuntimeExtension(
@@ -92,13 +109,23 @@ public class ObsTransferEndToEndTest {
     private String consumerEndpoint;
     private String providerEndpoint;
 
+    private final TypeTransformerRegistry registry = new TypeTransformerRegistryImpl();
+    private ObjectMapper mapper;
+
     @BeforeEach
     void setup() {
         consumerEndpoint = "http://localhost:%s".formatted(consumerContainer.getMappedPort(MINIO_CONTAINER_PORT));
         providerEndpoint = "http://localhost:%s".formatted(providerContainer.getMappedPort(MINIO_CONTAINER_PORT));
 
         providerClient = TestFunctions.createClient(PROVIDER_AK, PROVIDER_SK, providerEndpoint);
         consumerClient = TestFunctions.createClient(CONSUMER_AK, CONSUMER_SK, consumerEndpoint);
+
+        var builderFactory = Json.createBuilderFactory(Map.of());
+        mapper = JacksonJsonLd.createObjectMapper();
+        registry.register(new JsonObjectFromDataFlowStartMessageTransformer(builderFactory, mapper));
+        registry.register(new JsonObjectFromDataAddressDspaceTransformer(builderFactory, mapper));
+        registry.register(new JsonObjectToDataAddressDspaceTransformer());
+        registry.register(new JsonObjectToDataFlowResponseMessageTransformer());
     }
 
     @Test
@@ -114,12 +141,14 @@ void transfer_singleFile() throws IOException {
         consumerClient.createBucket(destBucket);
 
         var flowRequest = createFlowRequest(destBucket, consumerEndpoint, srcBucket, TESTFILE_NAME, providerEndpoint).build();
-        var url = PROVIDER.getControlEndpoint().getUrl().toString() + "/transfer";
+        var url = PROVIDER.getControlEndpoint().getUrl().toString() + "/v1/dataflows";
+
+        var startMessage = registry.transform(flowRequest, JsonObject.class).orElseThrow(failTest());
 
         given().when()
                 .baseUri(url)
                 .contentType(ContentType.JSON)
-                .body(flowRequest)
+                .body(startMessage)
                 .post()
                 .then()
                 .statusCode(200)
@@ -148,17 +177,22 @@ void transfer_multipleFilesWithPrefix() throws IOException {
         consumerClient.createBucket(destBucket);
 
         var flowRequest = createFlowRequest(destBucket, consumerEndpoint, srcBucket, "file", providerEndpoint).build();
-        var url = PROVIDER.getControlEndpoint().getUrl().toString() + "/transfer";
+        var url = PROVIDER.getControlEndpoint().getUrl().toString() + "/v1/dataflows";
 
-        given().when()
+
+        var startMessage = registry.transform(flowRequest, JsonObject.class).orElseThrow(failTest());
+
+        var response = given().when()
                 .baseUri(url)
                 .contentType(ContentType.JSON)
-                .body(flowRequest)
+                .body(startMessage)
                 .post()
                 .then()
                 .statusCode(200)
                 .log().all(true);
 
+        System.out.println(response);
+
         await().pollInterval(Duration.ofSeconds(2))
                 .atMost(Duration.ofSeconds(60))
                 .untilAsserted(() -> assertThat(consumerClient.listObjects(destBucket).getObjects())
@@ -170,18 +204,23 @@ void transfer_multipleFilesWithPrefix() throws IOException {
     }
 
     private DataFlowStartMessage.Builder createFlowRequest(String consumerBucket, String consumerEndpoint, String providerBucket, String providerObjectKey, String providerEndpoint) {
+        var id = UUID.randomUUID().toString();
         return DataFlowStartMessage.Builder.newInstance()
-                .id("test-request")
+                .processId("processId-" + id)
                 .sourceDataAddress(DataAddress.Builder.newInstance()
-                        .type(ObsBucketSchema.TYPE)
-                        .keyName(TESTFILE_NAME)
-                        .property(ObsBucketSchema.BUCKET_NAME, providerBucket)
-                        .property(ObsBucketSchema.KEY_PREFIX, providerObjectKey)
-                        .property(ObsBucketSchema.ACCESS_KEY_ID, PROVIDER_AK)
-                        .property(ObsBucketSchema.SECRET_ACCESS_KEY, PROVIDER_SK)
-                        .property(ObsBucketSchema.ENDPOINT, providerEndpoint)
-                        .build()
-                )
+                                .type(ObsBucketSchema.TYPE)
+                                .keyName(TESTFILE_NAME)
+                                .property(ObsBucketSchema.BUCKET_NAME, providerBucket)
+                                .property(ObsBucketSchema.KEY_PREFIX, providerObjectKey)
+                                .property(ObsBucketSchema.ACCESS_KEY_ID, PROVIDER_AK)
+                                .property(ObsBucketSchema.SECRET_ACCESS_KEY, PROVIDER_SK)
+                                .property(ObsBucketSchema.ENDPOINT, providerEndpoint)
+                                .build())
+                .transferType(new TransferType(ObsBucketSchema.TYPE, FlowType.PUSH))
+                .participantId("participantId")
+                .assetId("assetId")
+                .callbackAddress(URI.create("http://localhost"))
+                .agreementId("agreementId")
                 .destinationDataAddress(DataAddress.Builder.newInstance()
                         .type(ObsBucketSchema.TYPE)
                         .keyName(TESTFILE_NAME)
@@ -190,8 +229,12 @@ private DataFlowStartMessage.Builder createFlowRequest(String consumerBucket, St
                         .property(ObsBucketSchema.SECRET_ACCESS_KEY, CONSUMER_SK)
                         .property(ObsBucketSchema.ENDPOINT, consumerEndpoint)
                         .build()
-                )
-                .processId("test-process-id").flowType(FlowType.PUSH);
+                );
+    }
+
+    @NotNull
+    private Function<Failure, AssertionError> failTest() {
+        return f -> new AssertionError(f.getFailureDetail());
     }
 
 }
diff --git a/.../src/main/java/com/huawei/cloud/provision/obs/ObsConsumerResourceDefinitionGenerator.java b/.../src/main/java/com/huawei/cloud/provision/obs/ObsConsumerResourceDefinitionGenerator.java
@@ -47,6 +47,10 @@ public class ObsConsumerResourceDefinitionGenerator implements ConsumerResourceD
 
     @Override
     public boolean canGenerate(TransferProcess dataRequest, Policy policy) {
+        if (dataRequest.getDataDestination() == null) {
+            return false;
+        }
+
         return ObsBucketSchema.TYPE.equals(dataRequest.getDestinationType());
     }
 }