feat: add azure dataplane

.github/workflows/verify.yaml

@@ -152,6 +152,17 @@ jobs:
       - name: Run Postgresql E2E tests
         run: ./gradlew test -DincludeTags="PostgresqlIntegrationTest" -PverboseTest=true
 
+  dataplane-tests:
+    runs-on: ubuntu-latest
+    needs: [ verify-formatting, verify-license-headers ]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-java
+
+      - name: Run Azure/S3 dataplane tests
+        run: ./gradlew -p edc-tests/edc-dataplane test -DincludeTags="AzureCosmosDbIntegrationTest,AwsS3IntegrationTest"
+
   miw-integration-tests:
     runs-on: ubuntu-latest
     needs: [ verify-formatting, verify-license-headers ]
@@ -183,7 +194,7 @@ jobs:
           docker exec docker-environment-postgres-1 /opt/seed.sh
 
       - name: Run MIW Integration tests
-        run: | 
+        run: |
           ./gradlew -p edc-tests/miw-tests test -DincludeTags="MiwIntegrationTest" -PverboseTest=true
       - name: Run SSI E2E tests
         run: |

DEPENDENCIES

@@ -1,4 +1,4 @@
 maven/mavencentral/com.apicatalog/carbon-did/0.0.2, Apache-2.0, approved, #9239
 maven/mavencentral/com.apicatalog/iron-verifiable-credentials/0.8.1, Apache-2.0, approved, #9234
 maven/mavencentral/com.apicatalog/titanium-json-ld/1.0.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.apicatalog/titanium-json-ld/1.3.1, Apache-2.0, approved, #8912
@@ -10,8 +10,10 @@
 maven/mavencentral/com.azure/azure-identity/1.10.0, MIT AND Apache-2.0, approved, #10086
 maven/mavencentral/com.azure/azure-identity/1.9.2, MIT AND Apache-2.0, approved, #9686
 maven/mavencentral/com.azure/azure-json/1.0.1, MIT AND Apache-2.0, approved, #7933
-maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.6.4, MIT, approved, #7940
 maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.6.5, MIT, approved, #7940
+maven/mavencentral/com.azure/azure-storage-blob/12.23.0, MIT, approved, #9632
+maven/mavencentral/com.azure/azure-storage-common/12.22.0, MIT, approved, #9645
+maven/mavencentral/com.azure/azure-storage-internal-avro/12.8.0, MIT, approved, #9630
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.10.3, Apache-2.0, approved, CQ21280
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.13.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.13.5, Apache-2.0, approved, clearlydefined
@@ -28,6 +30,7 @@
 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.14.2, Apache-2.0, approved, #4105
 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.15.2, Apache-2.0, approved, #7934
 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-toml/2.15.2, Apache-2.0, approved, #9160
+maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.13.5, Apache-2.0, approved, #3768
 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2, Apache-2.0, approved, #4300
 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2, Apache-2.0, approved, #9237
 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.3, Apache-2.0, approved, #2566
@@ -213,6 +216,7 @@
 maven/mavencentral/org.eclipse.edc/auth-tokenbased/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/autodoc-processor/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/aws-s3-core/0.2.1, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/azure-blob-core/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/boot/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/callback-event-dispatcher/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/callback-http-dispatcher/0.2.1, Apache-2.0, approved, technology.edc
@@ -236,6 +240,7 @@
 maven/mavencentral/org.eclipse.edc/core-spi/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-api/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-aws-s3/0.2.1, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/data-plane-azure-storage/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-client/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-core/0.2.1, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-framework/0.2.1, Apache-2.0, approved, technology.edc

edc-dataplane/edc-dataplane-base/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
     runtimeOnly(libs.edc.config.filesystem)
     runtimeOnly(libs.edc.auth.tokenbased)
     runtimeOnly(libs.edc.dpf.awss3)
+    runtimeOnly(libs.edc.dpf.azblob)
     runtimeOnly(libs.edc.dpf.oauth2)
     runtimeOnly(libs.edc.dpf.http)
 

edc-tests/edc-dataplane/cloud-transfer-tests/build.gradle.kts

@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+plugins {
+    `java-library`
+}
+
+dependencies {
+
+    testImplementation(project(":edc-tests:e2e-tests"))
+    testImplementation(libs.edc.junit)
+    testImplementation(libs.restAssured)
+
+    // test runtime config
+    testImplementation(libs.edc.config.filesystem)
+    testImplementation(libs.edc.dpf.http)
+    testImplementation(libs.edc.auth.tokenbased)
+    testImplementation(libs.testcontainers.junit)
+    testImplementation(testFixtures(libs.edc.azure.test))
+    testImplementation(libs.edc.aws.s3.core)
+    testImplementation(testFixtures(libs.edc.aws.s3.test))
+    testImplementation(libs.aws.s3)
+    testImplementation(libs.aws.s3transfer)
+}
+
+
+

edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureBlobHelper.java

@@ -0,0 +1,95 @@
+/*
+ *
+ *   Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft
+ *
+ *   See the NOTICE file(s) distributed with this work for additional
+ *   information regarding copyright ownership.
+ *
+ *   This program and the accompanying materials are made available under the
+ *   terms of the Apache License, Version 2.0 which is available at
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *   License for the specific language governing permissions and limitations
+ *   under the License.
+ *
+ *   SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.eclipse.tractusx.edc.dataplane.transfer.test;
+
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.sas.BlobContainerSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.OffsetDateTime;
+import java.util.List;
+
+import static org.eclipse.edc.azure.testfixtures.TestFunctions.getBlobServiceClient;
+
+/**
+ * Helper class that internally uses Azure SDK classes to create containers, upload blobs, generate SAS tokens, etc.
+ */
+public class AzureBlobHelper {
+    private final String accountName;
+    private final String key;
+    private final String host;
+    private final int port;
+    private BlobServiceClient blobServiceClient;
+
+    public AzureBlobHelper(String accountName, String key, String host, int port) {
+        this.accountName = accountName;
+        this.key = key;
+        this.host = host;
+        this.port = port;
+    }
+
+    public BlobContainerClient createContainer(String containerName) {
+        return blobClient().createBlobContainer(containerName);
+    }
+
+    private BlobServiceClient blobClient() {
+        if (blobServiceClient == null) {
+            var endpoint = "http://%s:%s/%s".formatted(host, port, accountName);
+            blobServiceClient = getBlobServiceClient(accountName, key, endpoint);
+        }
+        return blobServiceClient;
+    }
+
+    public void uploadBlob(BlobContainerClient client, InputStream inputStream, String targetBlobName) {
+        client.getBlobClient(targetBlobName).upload(inputStream, true);
+    }
+
+    public void uploadBlob(BlobContainerClient client, String resourceName) {
+        try (var is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName)) {
+            client.getBlobClient(resourceName).upload(is, true);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public List<String> listBlobs(String container) {
+        if (blobClient().listBlobContainers().stream().noneMatch(bci -> bci.getName().equalsIgnoreCase(container))) {
+            return List.of();
+        }
+        return blobClient()
+                .getBlobContainerClient(container)
+                .listBlobs()
+                .stream().map(BlobItem::getName)
+                .toList();
+    }
+
+    public String generateAccountSas(String containerName) {
+        var expiry = OffsetDateTime.MAX.minusDays(1);
+        var permissions = BlobContainerSasPermission.parse("w");
+        var vals = new BlobServiceSasSignatureValues(expiry, permissions);
+        return blobClient().getBlobContainerClient(containerName).generateSas(vals);
+    }
+}