Skip to content

Commit

Permalink
ci(test): avoid use of deprecated class (#1629)
Browse files Browse the repository at this point in the history
* ci(test): avoid use of deprecated class

* dependencies

* refactor test pipeline
  • Loading branch information
ndr-brt authored Oct 17, 2024
1 parent 08c38d4 commit 8072f87
Show file tree
Hide file tree
Showing 11 changed files with 158 additions and 133 deletions.
34 changes: 10 additions & 24 deletions .github/workflows/verify.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,20 +100,23 @@ jobs:
strategy:
fail-fast: false
matrix:
variant: [ { dir: edc-tests/edc-controlplane/catalog-tests },
{ dir: edc-tests/edc-controlplane/edr-api-tests },
{ dir: edc-tests/edc-controlplane/iatp-tests },
{ dir: edc-tests/edc-controlplane/policy-tests },
{ dir: edc-tests/edc-controlplane/transfer-tests },
{ dir: edc-tests/edc-dataplane/edc-dataplane-tokenrefresh-tests }
variant: [
{ dir: edc-tests/edc-controlplane/catalog-tests },
{ dir: edc-tests/edc-controlplane/edr-api-tests },
{ dir: edc-tests/edc-controlplane/iatp-tests },
{ dir: edc-tests/edc-controlplane/policy-tests },
{ dir: edc-tests/edc-controlplane/transfer-tests },
{ dir: edc-tests/edc-dataplane/edc-dataplane-tokenrefresh-tests },
{ dir: edc-tests/edc-dataplane/cloud-transfer-tests },
{ dir: edc-tests/edc-end2end },
]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-java

- name: Run E2E tests (${{ matrix.variant.dir }})
run: |
./gradlew compileJava compileTestJava --refresh-dependencies
./gradlew compileJava compileTestJava
./gradlew -p ${{ matrix.variant.dir }} test -DincludeTags="EndToEndTest" -PverboseTest=true
postgres-tests:
Expand All @@ -124,20 +127,3 @@ jobs:

- name: Run Postgresql E2E tests
run: ./gradlew test -DincludeTags="PostgresqlIntegrationTest" --refresh-dependencies

cloud-transfer-tests:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-java

- name: Run Azure/S3 dataplane tests
run: |
./gradlew compileJava compileTestJava
./gradlew -p edc-tests/edc-dataplane test -DincludeTags="CloudTransferTest"
- name: Run Azure/S3 End-To-End tests
run: |
./gradlew compileJava compileTestJava
./gradlew -p edc-tests/edc-end2end test -DincludeTags="EndToEndTest"
18 changes: 9 additions & 9 deletions DEPENDENCIES
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,22 @@ maven/mavencentral/com.apicatalog/iron-verifiable-credentials/0.14.0, Apache-2.0
maven/mavencentral/com.apicatalog/titanium-json-ld/1.0.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/com.apicatalog/titanium-json-ld/1.4.0, Apache-2.0, approved, #15200
maven/mavencentral/com.apicatalog/titanium-json-ld/1.4.1, Apache-2.0, approved, #15200
maven/mavencentral/com.azure/azure-core-http-netty/1.15.2, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core-http-netty/1.15.3, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core-http-netty/1.15.4, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core-http-netty/1.15.5, , restricted, clearlydefined
maven/mavencentral/com.azure/azure-core-http-netty/1.15.2, MIT AND Apache-2.0, approved, #16697
maven/mavencentral/com.azure/azure-core-http-netty/1.15.3, MIT AND Apache-2.0, approved, #16697
maven/mavencentral/com.azure/azure-core-http-netty/1.15.4, MIT AND Apache-2.0, approved, #16697
maven/mavencentral/com.azure/azure-core-http-netty/1.15.5, MIT AND Apache-2.0, approved, #16697
maven/mavencentral/com.azure/azure-core/1.50.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core/1.51.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core/1.52.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-core/1.53.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-identity/1.13.2, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-identity/1.14.0, , restricted, clearlydefined
maven/mavencentral/com.azure/azure-identity/1.14.0, MIT, approved, #16714
maven/mavencentral/com.azure/azure-json/1.2.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-json/1.3.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.8.5, MIT, approved, #13690
maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.8.7, MIT, approved, #13690
maven/mavencentral/com.azure/azure-storage-blob/12.28.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-storage-blob/12.28.1, , restricted, clearlydefined
maven/mavencentral/com.azure/azure-storage-blob/12.28.1, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-storage-common/12.27.0, MIT, approved, clearlydefined
maven/mavencentral/com.azure/azure-storage-common/12.27.1, , restricted, clearlydefined
maven/mavencentral/com.azure/azure-storage-internal-avro/12.13.0, MIT, approved, clearlydefined
Expand Down Expand Up @@ -105,7 +105,7 @@ maven/mavencentral/com.lmax/disruptor/3.4.4, Apache-2.0, approved, clearlydefine
maven/mavencentral/com.microsoft.azure/msal4j-persistence-extension/1.3.0, MIT, approved, #14411
maven/mavencentral/com.microsoft.azure/msal4j/1.15.0, MIT, approved, clearlydefined
maven/mavencentral/com.microsoft.azure/msal4j/1.16.2, MIT, approved, clearlydefined
maven/mavencentral/com.microsoft.azure/msal4j/1.17.2, , restricted, clearlydefined
maven/mavencentral/com.microsoft.azure/msal4j/1.17.2, MIT, approved, #16709
maven/mavencentral/com.networknt/json-schema-validator/1.0.76, Apache-2.0, approved, CQ22638
maven/mavencentral/com.nimbusds/content-type/2.3, Apache-2.0, approved, clearlydefined
maven/mavencentral/com.nimbusds/lang-tag/1.7, Apache-2.0, approved, clearlydefined
Expand Down Expand Up @@ -636,8 +636,8 @@ maven/mavencentral/org.slf4j/slf4j-api/2.0.6, MIT, approved, #5915
maven/mavencentral/org.slf4j/slf4j-api/2.0.9, MIT, approved, #5915
maven/mavencentral/org.testcontainers/database-commons/1.20.2, Apache-2.0, approved, #16630
maven/mavencentral/org.testcontainers/jdbc/1.20.2, Apache-2.0, approved, #16621
maven/mavencentral/org.testcontainers/junit-jupiter/1.20.2, None, restricted, #16552
maven/mavencentral/org.testcontainers/postgresql/1.20.2, None, restricted, #16627
maven/mavencentral/org.testcontainers/junit-jupiter/1.20.2, MIT, approved, #16552
maven/mavencentral/org.testcontainers/postgresql/1.20.2, MIT, approved, #16627
maven/mavencentral/org.testcontainers/testcontainers/1.20.2, MIT, approved, #15747
maven/mavencentral/org.xmlresolver/xmlresolver/5.2.2, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.xmlunit/xmlunit-core/2.9.1, Apache-2.0, approved, #6272
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.restassured.http.ContentType;
import jakarta.json.Json;
import jakarta.json.JsonObjectBuilder;
import org.eclipse.edc.junit.annotations.EndToEndTest;
import org.eclipse.edc.junit.extensions.EmbeddedRuntime;
import org.eclipse.edc.junit.extensions.RuntimeExtension;
import org.eclipse.edc.junit.extensions.RuntimePerClassExtension;
Expand All @@ -35,7 +36,6 @@
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.testcontainers.containers.FixedHostPortGenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;

Expand All @@ -52,17 +52,13 @@
import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
import static org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage.EDC_DATA_FLOW_START_MESSAGE_TYPE;
import static org.eclipse.edc.util.io.Ports.getFreePort;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_KEY;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_NAME;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_CONTAINER_NAME;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_KEY_ALIAS;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_ACCOUNT_KEY;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_ACCOUNT_NAME;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_CONTAINER_NAME;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_KEY_ALIAS;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZURITE_CONTAINER_PORT;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZURITE_DOCKER_IMAGE;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.CONSUMER_AZURITE_ACCOUNT;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PREFIX_FOR_MUTIPLE_FILES;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.PROVIDER_AZURITE_ACCOUNT;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobDestinationAddress;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobSourceAddress;
Expand All @@ -79,7 +75,7 @@
*/
@SuppressWarnings("resource")
@Testcontainers
@CloudTransferTest
@EndToEndTest
public class AzureToAzureTest {

private static final int PROVIDER_CONTROL_PORT = getFreePort();
Expand All @@ -92,22 +88,22 @@ public class AzureToAzureTest {
RuntimeConfig.Azure.blobstoreDataplaneConfig("/control", PROVIDER_CONTROL_PORT, AZURITE_HOST_PORT),
":edc-tests:runtime:dataplane-cloud"
)).registerServiceMock(Monitor.class, spy(new ConsoleMonitor("AzureBlob-Dataplane", ConsoleMonitor.Level.DEBUG, true)));

/**
* Currently we have to use one container to host both consumer and provider accounts, because we cannot handle
* two different endpoint templates for provider and consumer. Endpoint templates are configured globally.
* Also, the host-port must be fixed/deterministic, as the {@code PROVIDER_RUNTIME} needs to know it in advance
*/
@Container
private final FixedHostPortGenericContainer<?> azuriteContainer = new FixedHostPortGenericContainer<>(AZURITE_DOCKER_IMAGE)
.withFixedExposedPort(AZURITE_HOST_PORT, AZURITE_CONTAINER_PORT)
.withEnv("AZURITE_ACCOUNTS", AZBLOB_PROVIDER_ACCOUNT_NAME + ":" + AZBLOB_PROVIDER_ACCOUNT_KEY + ";" + AZBLOB_CONSUMER_ACCOUNT_NAME + ":" + AZBLOB_CONSUMER_ACCOUNT_KEY);
private final AzuriteContainer azuriteContainer = new AzuriteContainer(AZURITE_HOST_PORT, PROVIDER_AZURITE_ACCOUNT, CONSUMER_AZURITE_ACCOUNT);

private AzureBlobHelper providerBlobHelper;
private AzureBlobHelper consumerBlobHelper;

@BeforeEach
void setup() {
providerBlobHelper = new AzureBlobHelper(AZBLOB_PROVIDER_ACCOUNT_NAME, AZBLOB_PROVIDER_ACCOUNT_KEY, azuriteContainer.getHost(), azuriteContainer.getMappedPort(AZURITE_CONTAINER_PORT));
consumerBlobHelper = new AzureBlobHelper(AZBLOB_CONSUMER_ACCOUNT_NAME, AZBLOB_CONSUMER_ACCOUNT_KEY, azuriteContainer.getHost(), azuriteContainer.getMappedPort(AZURITE_CONTAINER_PORT));
providerBlobHelper = azuriteContainer.getHelper(PROVIDER_AZURITE_ACCOUNT);
consumerBlobHelper = azuriteContainer.getHelper(CONSUMER_AZURITE_ACCOUNT);
}

@Test
Expand All @@ -120,7 +116,7 @@ void transferMultipleFile_success(Vault vault) {
fileNames.forEach(filename -> providerBlobHelper.uploadBlob(sourceContainer, fileData, filename));

consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key());
vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """
{"sas": "%s","edctype":"dataspaceconnector:azuretoken"}
""".formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME)));
Expand Down Expand Up @@ -157,7 +153,7 @@ void transferFile_success(Vault vault) {
// create container in consumer's blob store
consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME);

vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key());
vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """
{"sas": "%s","edctype":"dataspaceconnector:azuretoken"}
""".formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME)));
Expand Down Expand Up @@ -205,7 +201,7 @@ void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException {
// create container in consumer's blob store
consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME);

vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key());
vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """
{"sas": "%s","edctype":"dataspaceconnector:azuretoken"}
""".formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME)));
Expand All @@ -231,7 +227,7 @@ void transferFile_largeFile(long sizeBytes, Vault vault) throws IOException {
@Test
void transferFolder_targetFolderNotExists_shouldCreate(Vault vault) {

vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key());
var sas = consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME);
vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """
{"sas": "%s","edctype":"dataspaceconnector:azuretoken"}
Expand Down Expand Up @@ -275,7 +271,7 @@ void transferFile_targetContainerNotExist_shouldFail(Vault vault) {

providerBlobHelper.uploadBlob(sourceContainer, fileData, TESTFILE_NAME);

vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY);
vault.storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, PROVIDER_AZURITE_ACCOUNT.key());
vault.storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """
{"sas": "%s","edctype":"dataspaceconnector:azuretoken"}
""".formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME)));
Expand All @@ -292,7 +288,7 @@ void transferFile_targetContainerNotExist_shouldFail(Vault vault) {
await().pollInterval(Duration.ofSeconds(2))
.atMost(Duration.ofSeconds(10))
.untilAsserted(() -> verify(DATAPLANE_RUNTIME.getService(Monitor.class))
.severe(contains("Error creating blob %s on account %s".formatted(TESTFILE_NAME, AZBLOB_CONSUMER_ACCOUNT_NAME)), isA(IOException.class)));
.severe(contains("Error creating blob %s on account %s".formatted(TESTFILE_NAME, CONSUMER_AZURITE_ACCOUNT.name())), isA(IOException.class)));
}

private JsonObjectBuilder createFlowRequestBuilder(String blobName) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright (c) 2024 Cofinity-X
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/

package org.eclipse.tractusx.edc.dataplane.transfer.test;

import org.testcontainers.containers.GenericContainer;

import java.util.Arrays;
import java.util.List;

import static java.util.stream.Collectors.joining;
import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZURITE_CONTAINER_PORT;

public class AzuriteContainer extends GenericContainer<AzuriteContainer> {

public AzuriteContainer(int azuriteHostPort, Account... accounts) {
super(TestConstants.AZURITE_DOCKER_IMAGE);
var azuriteAccounts = Arrays.stream(accounts).map(it -> "%s:%s".formatted(it.name, it.key)).collect(joining(";"));
addEnv("AZURITE_ACCOUNTS", azuriteAccounts);
setPortBindings(List.of("%d:10000".formatted(azuriteHostPort)));
}

public AzureBlobHelper getHelper(Account account) {
return new AzureBlobHelper(account.name(), account.key(), getHost(), getMappedPort(AZURITE_CONTAINER_PORT));
}

public record Account(String name, String key) { }
}

This file was deleted.

Loading

0 comments on commit 8072f87

Please sign in to comment.