refactor: deprecate dataplane-selector-configuration in favor of data…

…-plane-self-registration
eclipse-tractusx · May 27, 2024 · e07562e · e07562e
1 parent 14f59fc
commit e07562e
Show file tree

Hide file tree

Showing 43 changed files with 109 additions and 228 deletions.
diff --git a/.github/actions/run-deployment-test/action.yml b/.github/actions/run-deployment-test/action.yml
@@ -84,6 +84,10 @@ runs:
       shell: bash
       run: ${{ inputs.helm_command }}
 
+    - name: Print logs
+      if: failure()
+      shell: bash
+      run: kubectl get deployments | tail -n +2 | awk '{print $1}' | sed 's/^/deployment\//' | xargs -n1 kubectl logs
 
     #################
     ### Tear Down ###

diff --git a/.github/workflows/upgradeability-test.yaml b/.github/workflows/upgradeability-test.yaml
@@ -117,6 +117,11 @@ jobs:
           # execute the helm test
           helm test tx-prod
 
+      - name: Print logs
+        if: failure()
+        shell: bash
+        run: kubectl get deployments | tail -n +2 | awk '{print $1}' | sed 's/^/deployment\//' | xargs -n1 kubectl logs
+
       - name: Destroy the kind cluster
         if: always()
         shell: bash

diff --git a/DEPENDENCIES b/DEPENDENCIES
@@ -392,8 +392,12 @@ maven/mavencentral/org.eclipse.edc/data-plane-http-oauth2/0.7.0, Apache-2.0, app
 maven/mavencentral/org.eclipse.edc/data-plane-http-spi/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-http/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-public-api-v2/0.7.0, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/data-plane-selector-api/0.7.0, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/data-plane-selector-client/0.7.0, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/data-plane-selector-control-api/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-selector-core/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-selector-spi/0.7.0, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/data-plane-self-registration/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-signaling-api-configuration/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-signaling-api/0.7.0, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-plane-signaling-client/0.7.0, Apache-2.0, approved, technology.edc

diff --git a/charts/tractusx-connector-azure-vault/templates/_helpers.tpl b/charts/tractusx-connector-azure-vault/templates/_helpers.tpl
@@ -137,12 +137,19 @@ Validation URL
 {{- end }}
 
 {{/*
-Control Plane URL
+Control Plane Control URL
 */}}
 {{- define "txdc.controlplane.url.control" -}}
 {{- printf "http://%s-controlplane:%v%s" ( include "txdc.fullname" $ ) $.Values.controlplane.endpoints.control.port $.Values.controlplane.endpoints.control.path -}}
 {{- end }}
 
+{{/*
+Data Plane Control URL
+*/}}
+{{- define "txdc.dataplane.url.control" -}}
+{{- printf "http://%s-dataplane:%v%s" ( include "txdc.fullname" $ ) $.Values.dataplane.endpoints.control.port $.Values.dataplane.endpoints.control.path -}}
+{{- end }}
+
 {{/*
 Data Signaling URL
 */}}

diff --git a/charts/tractusx-connector-azure-vault/templates/deployment-controlplane.yaml b/charts/tractusx-connector-azure-vault/templates/deployment-controlplane.yaml
@@ -298,25 +298,6 @@ spec:
               value: {{ .Values.controlplane.bdrs.cache_validity_seconds | quote}}
             {{- end}}
 
-            ################
-            ## DATA PLANE ##
-            ################
-
-            # see extension https://github.com/eclipse-tractusx/tractusx-edc/tree/main/edc-extensions/dataplane-selector-configuration
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_URL"
-              value: {{ include "txdc.dataplane.url.signaling" . }}/v1/dataflows
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_TRANSFERTYPES"
-              value: "HttpData-PULL,HttpData-PUSH,AmazonS3-PUSH,AzureStorage-PUSH"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_SOURCETYPES"
-              value: "HttpData,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_DESTINATIONTYPES"
-              value: "HttpData,HttpProxy,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_PROPERTIES"
-              value: |-
-                {{ printf "{ \"publicApiUrl\": \"%s\" }" (include "txdc.dataplane.url.public" . ) }}
-
-
-
             ###########
             ## VAULT ##
             ###########
@@ -352,7 +333,7 @@ spec:
             ######################################
             ## Additional environment variables ##
             ######################################
-            - name: "EDC_CONNECTOR_NAME"
+            - name: "EDC_RUNTIME_ID"
               value: {{ include "txdc.fullname" .}}-controlplane
           {{- range $key, $value := .Values.controlplane.envValueFrom }}
             - name: {{ $key | quote }}

diff --git a/charts/tractusx-connector-azure-vault/templates/deployment-dataplane.yaml b/charts/tractusx-connector-azure-vault/templates/deployment-dataplane.yaml
@@ -169,8 +169,10 @@ spec:
               value: {{ .Values.dataplane.endpoints.public.port | quote }}
             - name: "WEB_HTTP_PUBLIC_PATH"
               value: {{ .Values.dataplane.endpoints.public.path | quote }}
-            - name: "EDC_DATAPLANE_TOKEN_VALIDATION_ENDPOINT"
-              value: {{ include "txdc.controlplane.url.validation" .}}
+            - name: "EDC_CONTROL_ENDPOINT"
+              value: {{ include "txdc.dataplane.url.control" . }}
+            - name: "EDC_DPF_SELECTOR_URL"
+              value: {{ include "txdc.controlplane.url.control" . }}/v1/dataplanes
 
             #######
             # AWS #
@@ -296,7 +298,7 @@ spec:
             ######################################
             ## Additional environment variables ##
             ######################################
-            - name: "EDC_CONNECTOR_NAME"
+            - name: "EDC_RUNTIME_ID"
               value: {{ include "txdc.fullname" .}}-dataplane
           {{- range $key, $value := .Values.dataplane.envValueFrom }}
             - name: {{ $key | quote }}

diff --git a/charts/tractusx-connector-azure-vault/templates/service-dataplane.yaml b/charts/tractusx-connector-azure-vault/templates/service-dataplane.yaml
@@ -36,6 +36,10 @@ spec:
       targetPort: signaling
       protocol: TCP
       name: signaling
+    - port: {{ .Values.dataplane.endpoints.control.port }}
+      targetPort: control
+      protocol: TCP
+      name: control
     - port: {{ .Values.dataplane.endpoints.public.port }}
       targetPort: public
       protocol: TCP

diff --git a/charts/tractusx-connector-memory/templates/deployment-runtime.yaml b/charts/tractusx-connector-memory/templates/deployment-runtime.yaml
@@ -189,8 +189,6 @@ spec:
               value: {{ .Values.runtime.endpoints.public.port | quote }}
             - name: "WEB_HTTP_PUBLIC_PATH"
               value: {{ .Values.runtime.endpoints.public.path | quote }}
-            - name: "EDC_DATAPLANE_TOKEN_VALIDATION_ENDPOINT"
-              value: {{ include "txdc.runtime.url.validation" .}}
 
             #########
             ## DSP ##
@@ -236,21 +234,6 @@ spec:
             ################
             ## DATA PLANE ##
             ################
-
-            # see extension https://github.com/eclipse-tractusx/tractusx-edc/tree/develop/edc-extensions/dataplane-selector-configuration
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_URL"
-              value: {{ include "txdc.dataplane.url.signaling" . }}/v1/dataflows
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_TRANSFERTYPES"
-              value: "HttpData-PULL,HttpData-PUSH,AmazonS3-PUSH,AzureStorage-PUSH"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_SOURCETYPES"
-              value: "HttpData,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_DESTINATIONTYPES"
-              value: "HttpData,HttpProxy,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_PROPERTIES"
-              value: |-
-                {{ printf "{ \"publicApiUrl\": \"%s\" }" (include "txdc.dataplane.url.public" . ) }}
-
-
             - name: "EDC_DATAPLANE_API_PUBLIC_BASEURL"
               value: {{ include "txdc.dataplane.url.public" . }}
 

diff --git a/charts/tractusx-connector/templates/_helpers.tpl b/charts/tractusx-connector/templates/_helpers.tpl
@@ -136,12 +136,19 @@ Validation URL
 {{- end }}
 
 {{/*
-Control Plane URL
+Control Plane Control URL
 */}}
 {{- define "txdc.controlplane.url.control" -}}
 {{- printf "http://%s-controlplane:%v%s" ( include "txdc.fullname" $ ) $.Values.controlplane.endpoints.control.port $.Values.controlplane.endpoints.control.path -}}
 {{- end }}
 
+{{/*
+Data Plane Control URL
+*/}}
+{{- define "txdc.dataplane.url.control" -}}
+{{- printf "http://%s-dataplane:%v%s" ( include "txdc.fullname" $ ) $.Values.dataplane.endpoints.control.port $.Values.dataplane.endpoints.control.path -}}
+{{- end }}
+
 {{/*
 Data Signaling URL
 */}}

diff --git a/charts/tractusx-connector/templates/deployment-controlplane.yaml b/charts/tractusx-connector/templates/deployment-controlplane.yaml
@@ -299,26 +299,6 @@ spec:
               value: {{ .Values.controlplane.bdrs.cache_validity_seconds | quote}}
             {{- end}}
 
-            ################
-            ## DATA PLANE ##
-            ################
-
-            # see extension https://github.com/eclipse-tractusx/tractusx-edc/tree/main/edc-extensions/dataplane-selector-configuration
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_URL"
-              value: {{ include "txdc.dataplane.url.signaling" . }}/v1/dataflows
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_TRANSFERTYPES"
-              value: "HttpData-PULL,HttpData-PUSH,AmazonS3-PUSH,AzureStorage-PUSH"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_SOURCETYPES"
-              value: "HttpData,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_DESTINATIONTYPES"
-              value: "HttpData,HttpProxy,AmazonS3,AzureStorage"
-            - name: "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_PROPERTIES"
-              value: |-
-                {{ printf "{ \"publicApiUrl\": \"%s\" }" (include "txdc.dataplane.url.public" . ) }}
-
-
-
-
             ###########
             ## VAULT ##
             ###########
@@ -348,7 +328,7 @@ spec:
             ######################################
             ## Additional environment variables ##
             ######################################
-            - name: "EDC_CONNECTOR_NAME"
+            - name: "EDC_RUNTIME_ID"
               value: {{ include "txdc.fullname" .}}-controlplane
           {{- range $key, $value := .Values.controlplane.envValueFrom }}
             - name: {{ $key | quote }}

diff --git a/charts/tractusx-connector/templates/deployment-dataplane.yaml b/charts/tractusx-connector/templates/deployment-dataplane.yaml
@@ -169,8 +169,10 @@ spec:
               value: {{ .Values.dataplane.endpoints.public.port | quote }}
             - name: "WEB_HTTP_PUBLIC_PATH"
               value: {{ .Values.dataplane.endpoints.public.path | quote }}
-            - name: "EDC_DATAPLANE_TOKEN_VALIDATION_ENDPOINT"
-              value: {{ include "txdc.controlplane.url.validation" .}}
+            - name: "EDC_CONTROL_ENDPOINT"
+              value: {{ include "txdc.dataplane.url.control" . }}
+            - name: "EDC_DPF_SELECTOR_URL"
+              value: {{ include "txdc.controlplane.url.control" . }}/v1/dataplanes
 
             #######
             # AWS #
@@ -291,7 +293,7 @@ spec:
             ######################################
             ## Additional environment variables ##
             ######################################
-            - name: "EDC_CONNECTOR_NAME"
+            - name: "EDC_RUNTIME_ID"
               value: {{ include "txdc.fullname" .}}-dataplane
           {{- range $key, $value := .Values.dataplane.envValueFrom }}
             - name: {{ $key | quote }}

diff --git a/charts/tractusx-connector/templates/service-dataplane.yaml b/charts/tractusx-connector/templates/service-dataplane.yaml
@@ -40,6 +40,10 @@ spec:
       targetPort: signaling
       protocol: TCP
       name: signaling
+    - port: {{ .Values.dataplane.endpoints.control.port }}
+      targetPort: control
+      protocol: TCP
+      name: control
     - port: {{ .Values.dataplane.endpoints.public.port }}
       targetPort: public
       protocol: TCP

diff --git a/edc-controlplane/edc-controlplane-base/build.gradle.kts b/edc-controlplane/edc-controlplane-base/build.gradle.kts
@@ -25,7 +25,6 @@ plugins {
 dependencies {
     runtimeOnly(project(":core:edr-core"))
     runtimeOnly(project(":edc-extensions:dataplane:dataplane-selector-configuration"))
-
     runtimeOnly(project(":edc-extensions:provision-additional-headers"))
     runtimeOnly(project(":edc-extensions:edr:edr-api-v2"))
     runtimeOnly(project(":edc-extensions:edr:edr-callback"))
@@ -58,6 +57,7 @@ dependencies {
     runtimeOnly(libs.edc.config.filesystem)
     runtimeOnly(libs.edc.auth.tokenbased)
     runtimeOnly(libs.edc.validator.data.address.http.data)
+    runtimeOnly(libs.edc.data.plane.selector.control.api)
 
     runtimeOnly(libs.edc.api.management)
     runtimeOnly(libs.edc.api.controlplane)

diff --git a/edc-controlplane/edc-runtime-memory/build.gradle.kts b/edc-controlplane/edc-runtime-memory/build.gradle.kts
@@ -27,7 +27,9 @@ dependencies {
     implementation(project(":core:core-utils"))
     implementation(libs.edc.spi.core)
     runtimeOnly(project(":edc-controlplane:edc-controlplane-base"))
-    runtimeOnly(project(":edc-dataplane:edc-dataplane-base"))
+    runtimeOnly(project(":edc-dataplane:edc-dataplane-base")) {
+        exclude("org.eclipse.edc", "data-plane-selector-client")
+    }
     runtimeOnly(libs.edc.core.controlplane)
     testImplementation(libs.edc.junit)
     testImplementation(libs.edc.lib.boot)

diff --git a/edc-dataplane/edc-dataplane-base/build.gradle.kts b/edc-dataplane/edc-dataplane-base/build.gradle.kts
@@ -34,7 +34,6 @@ dependencies {
     runtimeOnly(libs.edc.core.did) // for the DID Public Key Resolver
     runtimeOnly(libs.edc.identity.did.web)
     runtimeOnly(libs.edc.config.filesystem)
-    runtimeOnly(libs.edc.auth.tokenbased)
     runtimeOnly(libs.edc.dpf.awss3)
     runtimeOnly(libs.edc.dpf.azblob)
     runtimeOnly(libs.edc.dpf.oauth2)
@@ -43,7 +42,8 @@ dependencies {
     runtimeOnly(libs.edc.dpf.core)
     runtimeOnly(libs.edc.controlplane.apiclient)
 
-    runtimeOnly(libs.edc.dpf.api.control)
+    runtimeOnly(libs.edc.data.plane.selector.client)
+    runtimeOnly(libs.edc.data.plane.self.registration)
     runtimeOnly(libs.edc.dpf.api.signaling)
 
     runtimeOnly(libs.edc.dpf.api.public.v2)

diff --git a/.../edc/dataplane/selector/configuration/DataPlaneSelectorConfigurationServiceExtension.java b/.../edc/dataplane/selector/configuration/DataPlaneSelectorConfigurationServiceExtension.java
@@ -40,6 +40,7 @@
 import static java.lang.String.format;
 import static java.lang.String.join;
 
+@Deprecated(since = "0.7.2")
 @Requires({ DataPlaneSelectorService.class })
 public class DataPlaneSelectorConfigurationServiceExtension implements ServiceExtension {
 
@@ -51,7 +52,7 @@ public class DataPlaneSelectorConfigurationServiceExtension implements ServiceEx
     public static final String PROPERTIES_SUFFIX = "properties";
     public static final String PUBLIC_API_URL_PROPERTY = "publicApiUrl";
 
-    private static final String NAME = "Data Plane Selector Configuration Extension";
+    private static final String NAME = "DEPRECATED: Data Plane Selector Configuration Extension";
     private static final String COMMA = ",";
     private static final String LOG_MISSING_CONFIGURATION = NAME + ": Missing configuration for " + CONFIG_PREFIX + ".%s.%s";
     private static final String LOG_SKIP_BC_MISSING_CONFIGURATION = NAME + ": Configuration issues. Skip registering of Data Plane Instance '%s'";
@@ -66,19 +67,24 @@ public String name() {
     }
 
     @Override
-    public void initialize(final ServiceExtensionContext serviceExtensionContext) {
-        this.dataPlaneSelectorService =
-                serviceExtensionContext.getService(DataPlaneSelectorService.class);
-        this.monitor = serviceExtensionContext.getMonitor();
+    public void initialize(ServiceExtensionContext context) {
+        this.dataPlaneSelectorService = context.getService(DataPlaneSelectorService.class);
+        this.monitor = context.getMonitor();
 
-        var config = serviceExtensionContext.getConfig(CONFIG_PREFIX);
+        var config = context.getConfig(CONFIG_PREFIX);
 
         config.partition().forEach(this::configureDataPlaneInstance);
     }
 
     private void configureDataPlaneInstance(final Config config) {
         var id = config.currentNode();
 
+        monitor.warning("""
+                The dataplane-selector-configuration has been deprecated, as now data-planes can register themselves
+                through the data-plane-self-registration extension. Please remove the 'edc.dataplane.selector.%s' config
+                group and configure your data-plane accordingly
+                """.formatted(id));
+
         var url = config.getString(URL_SUFFIX, "");
         var sourceTypes =
                 Arrays.stream(config.getString(SOURCE_TYPES_SUFFIX, "").split(COMMA))

diff --git a/.../dataplane/selector/configuration/DataPlaneSelectorConfigurationServiceExtensionTest.java b/.../dataplane/selector/configuration/DataPlaneSelectorConfigurationServiceExtensionTest.java
@@ -38,15 +38,14 @@
 import java.util.Map;
 import java.util.stream.Stream;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.CONFIG_PREFIX;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.DESTINATION_TYPES_SUFFIX;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.PROPERTIES_SUFFIX;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.PUBLIC_API_URL_PROPERTY;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.SOURCE_TYPES_SUFFIX;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.TRANSFER_TYPES_SUFFIX;
 import static org.eclipse.tractusx.edc.dataplane.selector.configuration.DataPlaneSelectorConfigurationServiceExtension.URL_SUFFIX;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.anyString;
 import static org.mockito.Mockito.argThat;
@@ -87,8 +86,7 @@ void setup() {
     void testName() {
         var extension = new DataPlaneSelectorConfigurationServiceExtension();
 
-        assertNotNull(extension.name());
-        assertEquals("Data Plane Selector Configuration Extension", extension.name());
+        assertThat(extension.name()).contains("Data Plane Selector Configuration Extension");
     }
 
     @Test
@@ -124,8 +122,8 @@ void testWarningOnPropertyMissing(String configKey, String configValue) {
         when(serviceExtensionContext.getConfig(CONFIG_PREFIX)).thenReturn(config);
         extension.initialize(serviceExtensionContext);
 
-        // one warning config missing, one warning data plane instance skipped
-        verify(monitor, times(2)).warning(anyString());
+        // one warning deprecation, one warning config missing, one warning data plane instance skipped
+        verify(monitor, times(3)).warning(anyString());
     }
 
     @Test

diff --git a/edc-tests/deployment/src/main/resources/helm/tractusx-connector-test.yaml b/edc-tests/deployment/src/main/resources/helm/tractusx-connector-test.yaml
@@ -61,6 +61,10 @@ controlplane:
     server:
       url: "https://bdrs.test.org"
 dataplane:
+  debug:
+    enabled: true
+    port: 1044
+    suspendOnStart: false
   endpoints:
     proxy:
       authKey: password