Version 2.3.0 (#3)

* feat: s3 endpoint regions validation (Digital-Ecosystems#80) * fix: regions endpoint signature changes (Digital-Ecosystems#82) Co-authored-by: Glaucio Jannotti <[email protected]> --------- Co-authored-by: Glaucio Jannotti <[email protected]> Co-authored-by: Glaucio Jannotti <[email protected]>
ionos-cloud · Dec 4, 2024 · 0366e9c · 0366e9c
1 parent 6f982f5
commit 0366e9c
Show file tree

Hide file tree

Showing 32 changed files with 203 additions and 237 deletions.
diff --git a/README.md b/README.md
@@ -59,15 +59,15 @@ The credentials can be found/configured in one of the following:
 
 It is required to configure those parameters:
 
-| Parameter name                       | Description                                                                                                                             | Mandatory                                            |
-|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
-| `edc.ionos.access.key`               | IONOS Access Key Id to access S3                                                                                                        | Yes if the context is accessing file                 |
-| `edc.ionos.secret.access.key`        | IONOS Secret Access Key to access S3                                                                                                    | Yes if the context is accessing file                 |
-| `edc.ionos.token`                    | IONOS token to allow S3 provisioning                                                                                                    | Yes if the context is provisioning access for others |
-| `edc.ionos.endpoint`                 | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information.                                                                | Yes, if the context is accessing file                | No, the default value is                             |
-| `edc.ionos.max.files`                | Maximum number of files retrieved by list files function.                                                                               | No, the default value is 5,000 files                 |
-| `edc.ionos.key.validation.attempts`  | Maximum number of attemps to validate a temporary key after its creation.                                                               | No, the default values is 10 attempts                |
-| `edc.ionos.key.validation.delay`     | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number.   | No, the default value is 3,000 (3 seconds)           |
+| Parameter name                      | Description                                                                                                                                        | Mandatory                                            |
+|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
+| `edc.ionos.access.key`              | IONOS Access Key Id to access S3                                                                                                                   | Yes if the context is accessing file                 |
+| `edc.ionos.secret.access.key`       | IONOS Secret Access Key to access S3                                                                                                               | Yes if the context is accessing file                 |
+| `edc.ionos.token`                   | IONOS token to allow S3 provisioning                                                                                                               | Yes if the context is provisioning access for others |
+| `edc.ionos.endpoint.region`         | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information.      | No, the default value is de                          |
+| `edc.ionos.max.files`               | Maximum number of files retrieved by list files function.                                                                                          | No, the default value is 5,000 files                 |
+| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation.                                                                          | No, the default values is 10 attempts                |
+| `edc.ionos.key.validation.delay`    | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number.              | No, the default value is 3,000 (3 seconds)           |
 
 To create the token please take a look at the following [documentation](./ionos_token.md).
 

diff --git a/assets.md b/assets.md
@@ -9,23 +9,23 @@ The asset registration aims to specify which file/folder we want to share. We ca
 ### Requirements
 
 
-| Parameter        | Description                                                                                                                                                                                                                                                                                                                                                                         | Mandatory |
-|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-| `storage`        | IONOS S3 endpoint address. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information.                                                                                                                                                                                                                                 | yes       |
-| `bucketName`     | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.                                                                                                                                                                                                                                 | yes       |
-| `blobName`       | File name or path to folder                                                                                                                                                                                                                                                                                                                                                         | yes       |
-| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer <br/> * do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied                                         | no        |
-| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer <br/> | no        |
+| Parameter        | Description                                                                                                                                                                                                                                                                                                                                 | Mandatory              |
+|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+| `region`        | IONOS S3 endpoint region. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information.                                                                                                                                                                                           | no, default value = de |
+| `bucketName`     | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.                                                                                                                                                                                                                                                                      | yes                    |
+| `blobName`       | File name or path to folder                                                                                                                                                                                                                                                                                                                 | yes                    |
+| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer <br/> * do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied | no                     |
+| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer <br/>                                                                                                                                                                       | no                     |
 
 Note:  if `filterIncludes` and  `filterExcludes` parameters are satisfied, the files to be copied will be selected using the `filterIncludes` and after that selected list, the files that have the pattern defined in the `filterExcludes` will be ignored.
 
 
 ## Example
 
 ```json
-"dataAddress":{
+"dataAddress": {
   "type": "IonosS3", //from EDC
-  "storage": "s3-eu-central-1.ionoscloud.com",
+  "region": "de,
   "bucketName": "mybucket",
   "blobName": "folder1/",
   "filterIncludes": "file1.csv",
@@ -40,19 +40,19 @@ The transfer of assets aims to transfer the files/folders from one connector to
 ### Requirements
 
 
-| Parameter    | Description                                                                                                                                         | Mandatory |
-|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-| `storage`    | IONOS S3 endpoint address. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information. | yes       |
-| `bucketName` | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes       |
-| `path`       | Path of destination where the file/folder will be placed. </br>  *if the path not filled, the file will be placed in the root of the bucket.        | no        |
+| Parameter    | Description                                                                                                                                        | Mandatory              |
+|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+| `region`     | IONOS S3 endpoint region. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information. | no, default value = de |
+| `bucketName` | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.                                                                             | yes                    |
+| `path`       | Path of destination where the file/folder will be placed. </br>  *if the path not filled, the file will be placed in the root of the bucket.       | no                     |
 
 
 ## Example
 
 ```json
-"dataDestination":{
+"dataDestination": {
   "type": "IonosS3", //from EDC
-  "storage": "s3-eu-central-1.ionoscloud.com",
+  "region": "de",
   "bucketName": "mybucket",
   "path": "folder2/",
   "keyName": "mykey" //from EDC

diff --git a/deployment/README.md b/deployment/README.md
@@ -48,7 +48,7 @@ export TF_VAR_persistence_type='PostgreSQLaaS' # 'PostgreSQLaaS', 'PostgreSQL' o
 export TF_VAR_vaultname='vault'  # optional if only 1 connector per cluster
 export TF_VAR_s3_access_key='' # S3 access key
 export TF_VAR_s3_secret_key='' # S3 secret key
-export TF_VAR_s3_endpoint='' # s3 endpoint (e.g. s3-eu-central-1.ionoscloud.com)
+export TF_VAR_s3_endpoint_region='' # s3 endpoint region (e.g. de)
 export TF_VAR_ionos_token='' # IONOS Cloud token, for further information: https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints
 
 # Required only if persistence_type is PostgreSQLaaS
@@ -86,7 +86,7 @@ In case you want to configure this Connector without Hashicorp Vault, you need t
 
 ```yaml
   ionos:
-    endpoint: <YOUR-S3-ENDPOINT>
+    region: <IONOS-S3-ENDPOINT-REGION>
     accessKey: <YOUR-KEY>
     secretKey: <YOUR-SECRET-KEY>
     token: <IONOS-TOKEN>

diff --git a/deployment/helm/README.md b/deployment/helm/README.md
@@ -35,7 +35,7 @@ The IONOS S3 Extension can be deployed to a Kubernetes cluster using the Helm ch
     # Add secrets to Vault
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.access.key content=
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.secret.key content=
-    kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint content=
+    kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content=
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.token content=
     ```
 

diff --git a/deployment/helm/edc-ionos-s3/templates/configmap.yaml b/deployment/helm/edc-ionos-s3/templates/configmap.yaml
@@ -18,7 +18,7 @@ data:
     edc.participant.id={{ .Values.edc.participant.id }}
     edc.ionos.access.key={{ .Values.edc.ionos.accessKey }}
     edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }}
-    edc.ionos.endpoint={{ .Values.edc.ionos.endpoint }}
+    edc.ionos.endpoint.region={{ .Values.edc.ionos.endpoint.region }}
     edc.ionos.token={{ .Values.edc.ionos.token }}
     edc.vault.hashicorp.url={{ .Values.edc.vault.hashicorp.url }}
     edc.vault.hashicorp.token={{ .Values.edc.vault.hashicorp.token }}

diff --git a/deployment/kind/README.md b/deployment/kind/README.md
@@ -26,7 +26,7 @@ Set environment variables
 export KUBECONFIG=path/to/kubeconfig
 export S3_ACCESS_KEY=''
 export S3_SECRET_KEY=''
-export S3_ENDPOINT=''
+export S3_ENDPOINT_REGION=''
 export IONOS_TOKEN=''
 ```
 

diff --git a/deployment/kind/scripts/deploy.sh b/deployment/kind/scripts/deploy.sh
@@ -34,8 +34,8 @@ if [ -z `printenv S3_SECRET_KEY` ]; then
     echo "Stopping because S3_SECRET_KEY is undefined"
     exit 1
 fi 
-if [ -z `printenv S3_ENDPOINT` ]; then
-    echo "Stopping because S3_ENDPOINT is undefined"
+if [ -z `printenv S3_ENDPOINT_REGION` ]; then
+    echo "Stopping because S3_ENDPOINT_REGION is undefined"
     exit 1
 fi
 if [ -z `printenv IONOS_TOKEN` ]; then
@@ -72,7 +72,7 @@ helm install -n edc-ionos-s3 --wait vault hashicorp/vault \
 export TF_VAR_kubeconfig=$KUBECONFIG
 export TF_VAR_s3_access_key=$S3_ACCESS_KEY
 export TF_VAR_s3_secret_key=$S3_SECRET_KEY
-export TF_VAR_s3_endpoint=$S3_ENDPOINT
+export TF_VAR_s3_endpoint_region=$S3_ENDPOINT_REGION
 export TF_VAR_ionos_token=$IONOS_TOKEN
 ../terraform/vault-init/vault-init.sh
 

diff --git a/deployment/terraform/deploy-services.sh b/deployment/terraform/deploy-services.sh
@@ -19,8 +19,8 @@ if [[ -z `printenv TF_VAR_s3_secret_key` ]]; then
     exit 1
 fi
 
-if [[ -z `printenv TF_VAR_s3_endpoint` ]]; then
-    echo "Stopping because TF_VAR_s3_endpoint is undefined"
+if [[ -z `printenv TF_VAR_s3_endpoint_region` ]]; then
+    echo "Stopping because TF_VAR_s3_endpoint_region is undefined"
     exit 1
 fi
 

diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf
@@ -56,9 +56,6 @@ variable "pg_password" {
   default = "postgres"
 }
 
-variable "s3_access_key" {}
-variable "s3_secret_key" {}
-variable "s3_endpoint" {}
 variable "ionos_token" {}
 
 variable "vaultname" {
@@ -98,8 +95,8 @@ resource "helm_release" "edc-ionos-s3" {
   }
 
   set {
-    name  = "edc.ionos.endpoint"
-    value = var.s3_endpoint
+    name  = "edc.ionos.endpoint.region"
+    value = var.s3_endpoint_region
   }
 
   set {

diff --git a/deployment/terraform/vault-init/vault-init.sh b/deployment/terraform/vault-init/vault-init.sh
@@ -39,5 +39,5 @@ fi
 # Add secrets to Vault
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.access.key content=$TF_VAR_s3_access_key
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.secret.key content=$TF_VAR_s3_secret_key
-kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint content=$TF_VAR_s3_endpoint
+kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint.region content=$TF_VAR_s3_endpoint_region
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.token content=$TF_VAR_ionos_token
diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts
@@ -9,7 +9,6 @@ val metaModelVersion: String by project
 val minIOVersion: String by project
 val extensionsGroup: String by project
 val extensionsVersion: String by project
-val junitVersion: String by project
 
 val gitHubPkgsName: String by project
 val gitHubPkgsUrl: String by project
@@ -21,20 +20,13 @@ dependencies {
 
 	implementation("${edcGroup}:transfer-spi:${edcVersion}")
 	implementation("io.minio:minio:${minIOVersion}")
-
-	testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
-	testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
 }
 
 java {
 	withJavadocJar()
 	withSourcesJar()
 }
 
-tasks.test {
-	useJUnitPlatform()
-}
-
 publishing {
 	publications {
 		create<MavenPublication>("maven") {

diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java
@@ -41,4 +41,5 @@ public interface S3ConnectorApi {
 
     void deleteAccessKey(String keyID);
 
+    S3ConnectorApi clone(String region, String accessKey, String secretKey);
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -41,4 +41,5 @@ public interface S3ConnectorApi {

		void deleteAccessKey(String keyID);

		S3ConnectorApi clone(String region, String accessKey, String secretKey);
		}