feat: add Management Domains feature (experimental) (#1408)

* feat: add FederatedCatalog modules * disable by default, add simple node directory * udpate other charts * added doc * helm docs, lint * fix tests * checkstyle * fix tests * DEPENDENCIES * DEPENDENCIES
eclipse-tractusx · Jul 11, 2024 · 75bdacb · 75bdacb
1 parent 51d3ef5
commit 75bdacb
Show file tree

Hide file tree

Showing 30 changed files with 697 additions and 20 deletions.
diff --git a/DEPENDENCIES b/DEPENDENCIES
@@ -363,6 +363,7 @@ maven/mavencentral/org.eclipse.edc/callback-static-endpoint/0.7.2-20240709-SNAPS
 maven/mavencentral/org.eclipse.edc/catalog-api/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/catalog-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/catalog-spi/0.7.2-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/catalog-util-lib/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/configuration-filesystem/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/connector-core/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/contract-agreement-api/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
@@ -387,6 +388,8 @@ maven/mavencentral/org.eclipse.edc/control-plane-transfer/0.7.2-20240709-SNAPSHO
 maven/mavencentral/org.eclipse.edc/control-plane-transform/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/core-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/core-spi/0.7.2-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/crawler-core/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/crawler-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/credential-query-lib/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/crypto-common-lib/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/data-address-http-data-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
@@ -434,6 +437,9 @@ maven/mavencentral/org.eclipse.edc/edr-index-sql/0.7.2-20240709-SNAPSHOT, Apache
 maven/mavencentral/org.eclipse.edc/edr-store-core/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/edr-store-receiver/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/edr-store-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/federated-catalog-api/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/federated-catalog-core/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc/federated-catalog-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/http-lib/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/http-spi/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/http/0.7.2-20240709-SNAPSHOT, Apache-2.0, approved, technology.edc

diff --git a/charts/tractusx-connector-azure-vault/README.md b/charts/tractusx-connector-azure-vault/README.md
@@ -74,10 +74,18 @@ helm install my-release tractusx-edc/tractusx-connector-azure-vault --version 0.
 | controlplane.autoscaling.targetMemoryUtilizationPercentage | int | `80` | targetAverageUtilization of memory provided to a pod |
 | controlplane.bdrs.cache_validity_seconds | int | `600` | Time that a cached BPN/DID resolution map is valid in seconds, default is 600 seconds (10 min) |
 | controlplane.bdrs.server.url | string | `nil` | URL of the BPN/DID Resolution Service |
+| controlplane.catalog | object | `{"crawler":{"initialDelay":null,"num":null,"period":null,"targetsFile":null},"enabled":false}` | configuration for the built-in federated catalog crawler |
+| controlplane.catalog.crawler.initialDelay | string | `nil` | Initial delay for the crawling to start. Leave blank for a random delay |
+| controlplane.catalog.crawler.num | string | `nil` | Number of desired crawlers. Final number might be different, based on number of crawl targets |
+| controlplane.catalog.crawler.period | string | `nil` | Period between two crawl runs in seconds. Default is 60 seconds. |
+| controlplane.catalog.crawler.targetsFile | string | `nil` | File path to a JSON file containing TargetNode entries |
+| controlplane.catalog.enabled | bool | `false` | Flag to globally enable/disable the FC feature |
 | controlplane.debug.enabled | bool | `false` | Enables java debugging mode. |
 | controlplane.debug.port | int | `1044` | Port where the debuggee can connect to. |
 | controlplane.debug.suspendOnStart | bool | `false` | Defines if the JVM should wait with starting the application until someone connected to the debugging port. |
-| controlplane.endpoints | object | `{"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"metrics":{"path":"/metrics","port":9090},"protocol":{"path":"/api/v1/dsp","port":8084}}` | endpoints of the control plane |
+| controlplane.endpoints | object | `{"catalog":{"path":"/catalog","port":8085},"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"metrics":{"path":"/metrics","port":9090},"protocol":{"path":"/api/v1/dsp","port":8084}}` | endpoints of the control plane |
+| controlplane.endpoints.catalog.path | string | `"/catalog"` | path for incoming catalog cache query requests |
+| controlplane.endpoints.catalog.port | int | `8085` | port for incoming catalog cache query requests |
 | controlplane.endpoints.control | object | `{"path":"/control","port":8083}` | control api, used for internal control calls. can be added to the internal ingress, but should probably not |
 | controlplane.endpoints.control.path | string | `"/control"` | path for incoming api calls |
 | controlplane.endpoints.control.port | int | `8083` | port for incoming api calls |

diff --git a/charts/tractusx-connector-azure-vault/templates/deployment-controlplane.yaml b/charts/tractusx-connector-azure-vault/templates/deployment-controlplane.yaml
@@ -172,6 +172,11 @@ spec:
               value: {{ .Values.controlplane.endpoints.protocol.path | quote }}
             - name: "EDC_CONTROL_ENDPOINT"
               value: {{ include "txdc.controlplane.url.control" .}}
+            - name: "WEB_HTTP_CATALOG_PORT"
+              value: {{ .Values.controlplane.endpoints.catalog.port | quote }}
+            - name: "WEB_HTTP_CATALOG_PATH"
+              value: {{ .Values.controlplane.endpoints.catalog.path | quote }}
+
 
             #########
             ## DSP ##
@@ -320,6 +325,30 @@ spec:
               value: {{ .Values.vault.azure.certificate | quote }}
             {{- end }}
 
+            ###############################
+            ## FEDERATED CATALOG CRAWLER ##
+            ###############################
+            {{- if .Values.controlplane.catalog.crawler.period }}
+            - name: "EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS"
+              value: {{ .Values.controlplane.catalog.crawler.period | quote}}
+            {{- end }}
+
+            {{- if .Values.controlplane.catalog.crawler.initialDelay }}
+            - name: "EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS"
+              value: {{ .Values.controlplane.catalog.crawler.initialDelay | quote }}
+            {{- end }}
+
+            {{- if .Values.controlplane.catalog.crawler.num }}
+            - name: "EDC_CATALOG_CACHE_PARTITION_NUM_CRAWLERS"
+              value: {{ .Values.controlplane.catalog.crawler.num }}
+            {{- end }}
+
+            - name: "EDC_CATALOG_CACHE_EXECUTION_ENABLED"
+              value: {{ .Values.controlplane.catalog.enabled | quote }}
+
+            - name: "TX_EDC_CATALOG_NODE_LIST_FILE"
+              value: {{ .Values.controlplane.catalog.crawler.targetsFile }}
+
             ######################################
             ## Additional environment variables ##
             ######################################

diff --git a/charts/tractusx-connector-azure-vault/values.yaml b/charts/tractusx-connector-azure-vault/values.yaml
@@ -141,13 +141,33 @@ controlplane:
       # -- path for incoming api calls
       path: /metrics
 
+    catalog:
+      # -- port for incoming catalog cache query requests
+      port: 8085
+      # -- path for incoming catalog cache query requests
+      path: /catalog
+
   bdrs:
     # -- Time that a cached BPN/DID resolution map is valid in seconds, default is 600 seconds (10 min)
     cache_validity_seconds: 600
     server:
       # -- URL of the BPN/DID Resolution Service
       url:
 
+  # -- configuration for the built-in federated catalog crawler
+  catalog:
+    # -- Flag to globally enable/disable the FC feature
+    enabled: false
+    crawler:
+      # -- Number of desired crawlers. Final number might be different, based on number of crawl targets
+      num:
+      # -- Period between two crawl runs in seconds. Default is 60 seconds.
+      period:
+      # -- Initial delay for the crawling to start. Leave blank for a random delay
+      initialDelay:
+      # -- File path to a JSON file containing TargetNode entries
+      targetsFile:
+
   service:
     # -- [Service type](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) to expose the running application on a set of Pods as a network service.
     type: ClusterIP
@@ -592,12 +612,12 @@ networkPolicy:
   controlplane:
     # -- Specify from rule network policy for cp (defaults to all namespaces)
     from:
-    - namespaceSelector: {}
+      - namespaceSelector: {}
   # -- Configuration of the dataplane component
   dataplane:
     # -- Specify from rule network policy for dp (defaults to all namespaces)
     from:
-    - namespaceSelector: {}
+      - namespaceSelector: {}
 
 serviceAccount:
   # -- Specifies whether a service account should be created

diff --git a/charts/tractusx-connector-memory/README.md b/charts/tractusx-connector-memory/README.md
@@ -74,10 +74,18 @@ helm install my-release tractusx-edc/tractusx-connector-memory --version 0.7.3 \
 | runtime.autoscaling.targetMemoryUtilizationPercentage | int | `80` | targetAverageUtilization of memory provided to a pod |
 | runtime.bdrs.cache_validity_seconds | int | `600` | Time that a cached BPN/DID resolution map is valid in seconds, default is 600 seconds (10 min) |
 | runtime.bdrs.server.url | string | `nil` | URL of the BPN/DID Resolution Service |
+| runtime.catalog | object | `{"crawler":{"initialDelay":null,"num":null,"period":null,"targetsFile":null},"enabled":false}` | configuration for the built-in federated catalog crawler |
+| runtime.catalog.crawler.initialDelay | string | `nil` | Initial delay for the crawling to start. Leave blank for a random delay |
+| runtime.catalog.crawler.num | string | `nil` | Number of desired crawlers. Final number might be different, based on number of crawl targets |
+| runtime.catalog.crawler.period | string | `nil` | Period between two crawl runs in seconds. Default is 60 seconds. |
+| runtime.catalog.crawler.targetsFile | string | `nil` | File path to a JSON file containing TargetNode entries |
+| runtime.catalog.enabled | bool | `false` | Flag to globally enable/disable the FC feature |
 | runtime.debug.enabled | bool | `false` | Enables java debugging mode. |
 | runtime.debug.port | int | `1044` | Port where the debuggee can connect to. |
 | runtime.debug.suspendOnStart | bool | `false` | Defines if the JVM should wait with starting the application until someone connected to the debugging port. |
-| runtime.endpoints | object | `{"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"protocol":{"path":"/api/v1/dsp","port":8084},"proxy":{"authKey":"password","path":"/proxy","port":8186},"public":{"path":"/api/public","port":8086}}` | endpoints of the controlplane |
+| runtime.endpoints | object | `{"catalog":{"path":"/catalog","port":8085},"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"protocol":{"path":"/api/v1/dsp","port":8084},"proxy":{"authKey":"password","path":"/proxy","port":8186},"public":{"path":"/api/public","port":8086}}` | endpoints of the controlplane |
+| runtime.endpoints.catalog.path | string | `"/catalog"` | path for incoming catalog cache query requests |
+| runtime.endpoints.catalog.port | int | `8085` | port for incoming catalog cache query requests |
 | runtime.endpoints.control | object | `{"path":"/control","port":8083}` | control api, used for internal control calls. can be added to the internal ingress, but should probably not |
 | runtime.endpoints.control.path | string | `"/control"` | path for incoming api calls |
 | runtime.endpoints.control.port | int | `8083` | port for incoming api calls |

diff --git a/charts/tractusx-connector-memory/templates/deployment-runtime.yaml b/charts/tractusx-connector-memory/templates/deployment-runtime.yaml
@@ -187,6 +187,10 @@ spec:
               value: {{ .Values.runtime.endpoints.public.port | quote }}
             - name: "WEB_HTTP_PUBLIC_PATH"
               value: {{ .Values.runtime.endpoints.public.path | quote }}
+            - name: "WEB_HTTP_CATALOG_PORT"
+              value: {{ .Values.runtime.endpoints.catalog.port | quote }}
+            - name: "WEB_HTTP_CATALOG_PATH"
+              value: {{ .Values.runtime.endpoints.catalog.path | quote }}
 
             #########
             ## DSP ##
@@ -270,13 +274,30 @@ spec:
             - name: "EDC_VAULT_SECRETS"
               value: {{ .Values.vault.secrets | quote}}
 
-            ###########################
-            ## AAS WRAPPER EXTENSION ##
-            ###########################
-            - name: "EDC_CP_ADAPTER_CACHE_CATALOG_EXPIRE_AFTER"
-              value: "0"
-            - name: "EDC_CP_ADAPTER_REUSE_CONTRACT_AGREEMENT"
-              value: "0"
+
+            ###############################
+            ## FEDERATED CATALOG CRAWLER ##
+            ###############################
+            {{- if .Values.runtime.catalog.crawler.period }}
+            - name: "EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS"
+              value: {{ .Values.runtime.catalog.crawler.period | quote}}
+            {{- end }}
+
+            {{- if .Values.runtime.catalog.crawler.initialDelay }}
+            - name: "EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS"
+              value: {{ .Values.runtime.catalog.crawler.initialDelay | quote }}
+            {{- end }}
+
+            {{- if .Values.runtime.catalog.crawler.num }}
+            - name: "EDC_CATALOG_CACHE_PARTITION_NUM_CRAWLERS"
+              value: {{ .Values.runtime.catalog.crawler.num }}
+            {{- end }}
+
+            - name: "EDC_CATALOG_CACHE_EXECUTION_ENABLED"
+              value: {{ .Values.runtime.catalog.enabled | quote }}
+
+            - name: "TX_EDC_CATALOG_NODE_LIST_FILE"
+              value: {{ .Values.runtime.catalog.crawler.targetsFile }}
 
             ######################################
             ## Additional environment variables ##

diff --git a/charts/tractusx-connector-memory/values.yaml b/charts/tractusx-connector-memory/values.yaml
@@ -144,6 +144,12 @@ runtime:
       # -- authentication key, must be attached to each request as `X-Api-Key` header
       authKey: "password"
 
+    catalog:
+      # -- port for incoming catalog cache query requests
+      port: 8085
+      # -- path for incoming catalog cache query requests
+      path: /catalog
+
   token:
     refresh:
       # -- TTL in seconds for access tokens (also known as EDR token)
@@ -166,6 +172,20 @@ runtime:
       # -- URL of the BPN/DID Resolution Service
       url:
 
+  # -- configuration for the built-in federated catalog crawler
+  catalog:
+    # -- Flag to globally enable/disable the FC feature
+    enabled: false
+    crawler:
+      # -- Number of desired crawlers. Final number might be different, based on number of crawl targets
+      num:
+      # -- Period between two crawl runs in seconds. Default is 60 seconds.
+      period:
+      # -- Initial delay for the crawling to start. Leave blank for a random delay
+      initialDelay:
+      # -- File path to a JSON file containing TargetNode entries
+      targetsFile:
+
   service:
     # -- [Service type](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) to expose the running application on a set of Pods as a network service.
     type: ClusterIP

diff --git a/charts/tractusx-connector/README.md b/charts/tractusx-connector/README.md
@@ -71,10 +71,18 @@ helm install my-release tractusx-edc/tractusx-connector --version 0.7.3 \
 | controlplane.autoscaling.targetMemoryUtilizationPercentage | int | `80` | targetAverageUtilization of memory provided to a pod |
 | controlplane.bdrs.cache_validity_seconds | int | `600` | Time that a cached BPN/DID resolution map is valid in seconds, default is 600 seconds (10 min) |
 | controlplane.bdrs.server.url | string | `nil` | URL of the BPN/DID Resolution Service |
+| controlplane.catalog | object | `{"crawler":{"initialDelay":null,"num":null,"period":null,"targetsFile":null},"enabled":false}` | configuration for the built-in federated catalog crawler |
+| controlplane.catalog.crawler.initialDelay | string | `nil` | Initial delay for the crawling to start. Leave blank for a random delay |
+| controlplane.catalog.crawler.num | string | `nil` | Number of desired crawlers. Final number might be different, based on number of crawl targets |
+| controlplane.catalog.crawler.period | string | `nil` | Period between two crawl runs in seconds. Default is 60 seconds. |
+| controlplane.catalog.crawler.targetsFile | string | `nil` | File path to a JSON file containing TargetNode entries |
+| controlplane.catalog.enabled | bool | `false` | Flag to globally enable/disable the FC feature |
 | controlplane.debug.enabled | bool | `false` | Enables java debugging mode. |
 | controlplane.debug.port | int | `1044` | Port where the debuggee can connect to. |
 | controlplane.debug.suspendOnStart | bool | `false` | Defines if the JVM should wait with starting the application until someone connected to the debugging port. |
-| controlplane.endpoints | object | `{"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"metrics":{"path":"/metrics","port":9090},"protocol":{"path":"/api/v1/dsp","port":8084}}` | endpoints of the control plane |
+| controlplane.endpoints | object | `{"catalog":{"path":"/catalog","port":8085},"control":{"path":"/control","port":8083},"default":{"path":"/api","port":8080},"management":{"authKey":"password","path":"/management","port":8081},"metrics":{"path":"/metrics","port":9090},"protocol":{"path":"/api/v1/dsp","port":8084}}` | endpoints of the control plane |
+| controlplane.endpoints.catalog.path | string | `"/catalog"` | path for incoming catalog cache query requests |
+| controlplane.endpoints.catalog.port | int | `8085` | port for incoming catalog cache query requests |
 | controlplane.endpoints.control | object | `{"path":"/control","port":8083}` | control api, used for internal control calls. can be added to the internal ingress, but should probably not |
 | controlplane.endpoints.control.path | string | `"/control"` | path for incoming api calls |
 | controlplane.endpoints.control.port | int | `8083` | port for incoming api calls |