From e62bf099ce0a2fabbd18a30cd98e32e0801933a5 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti <111659831+jannotti-glaucio@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:56:23 -0300 Subject: [PATCH] S3 Endpoint Regions Validation (#79) * feat: s3 endpoint regions validation * chore: fixing assets doc * fix: launchers config --- README.md | 18 +-- assets.md | 28 ++-- deployment/README.md | 6 +- deployment/helm/README.md | 2 +- .../edc-ionos-s3/templates/configmap.yaml | 2 +- deployment/helm/edc-ionos-s3/values.yaml | 2 +- deployment/kind/README.md | 2 +- deployment/kind/scripts/deploy.sh | 6 +- deployment/terraform/deploy-services.sh | 4 +- deployment/terraform/ionos-s3-deploy/main.tf | 6 +- deployment/terraform/vault-init/vault-init.sh | 2 +- extensions/core-ionos-s3/build.gradle.kts | 9 -- .../{configuration => }/S3CoreExtension.java | 28 ++-- .../ionosapi => api}/S3AccessKey.java | 16 ++- .../edc/extension/s3/api/S3ApiClient.java | 132 ++++++++++++++++++ .../ionos/edc/extension/s3/api/S3Region.java | 36 +++++ .../s3/connector/MinioConnector.java | 24 ---- .../S3Connector.java} | 8 +- .../S3ConnectorImpl.java} | 93 ++++++------ .../s3/connector/ionosapi/S3ApiConnector.java | 86 ------------ .../s3/schema/IonosBucketSchema.java | 4 +- .../s3/schema/IonosSettingsSchema.java | 2 +- .../{configuration => types}/IonosToken.java | 2 +- .../extension/s3/{api => types}/S3Object.java | 2 +- ...rg.eclipse.edc.spi.system.ServiceExtension | 2 +- .../s3/api/S3ConnectorApiImplTest.java | 80 ----------- .../ionos/s3/DataPlaneIonosS3Extension.java | 13 +- .../edc/dataplane/ionos/s3/IonosDataSink.java | 10 +- .../ionos/s3/IonosDataSinkFactory.java | 59 +++----- .../dataplane/ionos/s3/IonosDataSource.java | 25 ++-- .../ionos/s3/IonosDataSourceFactory.java | 10 +- .../IonosSourceDataAddressValidationRule.java | 2 +- .../ionos/s3/IonosDataSourceTest.java | 46 +++--- .../provision/s3/IonosProvisionExtension.java | 6 +- ...S3ConsumerResourceDefinitionGenerator.java | 4 +- .../s3/bucket/IonosS3ProvisionedResource.java | 4 +- .../s3/bucket/IonosS3Provisioner.java | 28 ++-- .../s3/bucket/IonosS3ResourceDefinition.java | 12 +- hashicorp/README.md | 4 +- .../resources/config.properties | 4 +- .../resources/config.properties | 3 +- .../resources/config.properties | 4 +- .../connector/resources/config.properties | 4 +- 43 files changed, 396 insertions(+), 444 deletions(-) rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{configuration => }/S3CoreExtension.java (68%) rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{connector/ionosapi => api}/S3AccessKey.java (65%) create mode 100644 extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java create mode 100644 extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java delete mode 100644 extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{api/S3ConnectorApi.java => connector/S3Connector.java} (80%) rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{api/S3ConnectorApiImpl.java => connector/S3ConnectorImpl.java} (67%) delete mode 100644 extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{configuration => types}/IonosToken.java (95%) rename extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/{api => types}/S3Object.java (95%) delete mode 100644 extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java diff --git a/README.md b/README.md index 4fef6bf0..472f32ac 100644 --- a/README.md +++ b/README.md @@ -59,15 +59,15 @@ The credentials can be found/configured in one of the following: It is required to configure those parameters: -| Parameter name | Description | Mandatory | -|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| -| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | -| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | -| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | -| `edc.ionos.endpoint` | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | Yes, if the context is accessing file | No, the default value is | -| `edc.ionos.max.files` | Maximum number of files retrieved by list files function. | No, the default value is 5,000 files | -| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation. | No, the default values is 10 attempts | -| `edc.ionos.key.validation.delay` | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number. | No, the default value is 3,000 (3 seconds) | +| Parameter name | Description | Mandatory | +|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| +| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | +| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | +| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | +| `edc.ionos.endpoint.region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | No, the default value is "de" | +| `edc.ionos.max.files` | Maximum number of files retrieved by list files function. | No, the default value is 5,000 files | +| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation. | No, the default values is 10 attempts | +| `edc.ionos.key.validation.delay` | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number. | No, the default value is 3,000 (3 seconds) | To create the token please take a look at the following [documentation](./ionos_token.md). diff --git a/assets.md b/assets.md index ec0fc54d..9af0d989 100644 --- a/assets.md +++ b/assets.md @@ -9,13 +9,13 @@ The asset registration aims to specify which file/folder we want to share. We ca ### Requirements -| Parameter | Description | Mandatory | -|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `storage` | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information. | yes | -| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes | -| `blobName` | File name or path to folder | yes | -| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer
* do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied | no | -| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer
| no | +| Parameter | Description | Mandatory | +|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information.| no, default value = de | +| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.| yes | +| `blobName` | File name or path to folder| yes | +| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer
* do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied| no | +| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer
| no | Note: if `filterIncludes` and `filterExcludes` parameters are satisfied, the files to be copied will be selected using the `filterIncludes` and after that selected list, the files that have the pattern defined in the `filterExcludes` will be ignored. @@ -25,7 +25,7 @@ Note: if `filterIncludes` and `filterExcludes` parameters are satisfied, the f ```json "dataAddress":{ "type": "IonosS3", //from EDC - "storage": "s3-eu-central-1.ionoscloud.com", + "region": "de", "bucketName": "mybucket", "blobName": "folder1/", "filterIncludes": "file1.csv", @@ -40,11 +40,11 @@ The transfer of assets aims to transfer the files/folders from one connector to ### Requirements -| Parameter | Description | Mandatory | -|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `storage` | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information. | yes | -| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes | -| `path` | Path of destination where the file/folder will be placed.
*if the path not filled, the file will be placed in the root of the bucket. | no | +| Parameter | Description | Mandatory | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information.| no, default value = de | +| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information| yes | +| `path` | Path of destination where the file/folder will be placed.
*if the path not filled, the file will be placed in the root of the bucket. | no | ## Example @@ -52,7 +52,7 @@ The transfer of assets aims to transfer the files/folders from one connector to ```json "dataDestination":{ "type": "IonosS3", //from EDC - "storage": "s3-eu-central-1.ionoscloud.com", + "region": "de", "bucketName": "mybucket", "path": "folder2/", "keyName": "mykey" //from EDC diff --git a/deployment/README.md b/deployment/README.md index 5cac45f7..8b6acd43 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -48,7 +48,7 @@ export TF_VAR_persistence_type='PostgreSQLaaS' # 'PostgreSQLaaS', 'PostgreSQL' o export TF_VAR_vaultname='vault' # optional if only 1 connector per cluster export TF_VAR_s3_access_key='' # S3 access key export TF_VAR_s3_secret_key='' # S3 secret key -export TF_VAR_s3_endpoint='' # s3 endpoint (e.g. s3-eu-central-1.ionoscloud.com) +export TF_VAR_s3_endpoint_region='' # s3 endpoint region (e.g. de) export TF_VAR_ionos_token='' # IONOS Cloud token, for further information: https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints # Required only if persistence_type is PostgreSQLaaS @@ -86,13 +86,13 @@ In case you want to configure this Connector without Hashicorp Vault, you need t ```yaml ionos: - endpoint: + region: accessKey: secretKey: token: ``` -They should be the same as the ones set in the environment variables. The **ionos.endpoint** is set to the default S3 location, but it can be changed to any other location. +They should be the same as the ones set in the environment variables. The **ionos.region** is set to the default S3 endpoint region, but it can be changed to any other location. If you don't want the Connector to be externally accessible, you need to set the following parameters in the helm [values.yaml](deployment/helm/edc-ionos-s3/values.yaml): diff --git a/deployment/helm/README.md b/deployment/helm/README.md index 86fb89ec..1ab673f1 100644 --- a/deployment/helm/README.md +++ b/deployment/helm/README.md @@ -35,7 +35,7 @@ The IONOS S3 Extension can be deployed to a Kubernetes cluster using the Helm ch # Add secrets to Vault kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.access.key content= kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.secret.key content= - kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint content= + kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content= kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.token content= ``` diff --git a/deployment/helm/edc-ionos-s3/templates/configmap.yaml b/deployment/helm/edc-ionos-s3/templates/configmap.yaml index d518cf68..e094843f 100644 --- a/deployment/helm/edc-ionos-s3/templates/configmap.yaml +++ b/deployment/helm/edc-ionos-s3/templates/configmap.yaml @@ -26,7 +26,7 @@ data: edc.vault.hashicorp.timeout.seconds={{ .Values.edc.vault.hashicorp.timeout.seconds }} edc.ionos.access.key={{ .Values.edc.ionos.accessKey }} edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }} - edc.ionos.endpoint={{ .Values.edc.ionos.endpoint }} + edc.ionos.endpoint.region={{ .Values.edc.ionos.region }} edc.ionos.token={{ .Values.edc.ionos.token }} {{- if eq .Values.edc.persistenceType "PostgreSQLaaS" }} diff --git a/deployment/helm/edc-ionos-s3/values.yaml b/deployment/helm/edc-ionos-s3/values.yaml index 13e5b172..d874e6ea 100644 --- a/deployment/helm/edc-ionos-s3/values.yaml +++ b/deployment/helm/edc-ionos-s3/values.yaml @@ -124,7 +124,7 @@ edc: timeout: seconds: 30 ionos: - endpoint: s3-eu-central-1.ionoscloud.com + region: de accessKey: notnull secretKey: notnull token: notnull diff --git a/deployment/kind/README.md b/deployment/kind/README.md index 43b31cd6..4b9db821 100644 --- a/deployment/kind/README.md +++ b/deployment/kind/README.md @@ -26,7 +26,7 @@ Set environment variables export KUBECONFIG=path/to/kubeconfig export S3_ACCESS_KEY='' export S3_SECRET_KEY='' -export S3_ENDPOINT='' +export S3_ENDPOINT_REGION='' export IONOS_TOKEN='' ``` diff --git a/deployment/kind/scripts/deploy.sh b/deployment/kind/scripts/deploy.sh index b04c954c..4ad48976 100755 --- a/deployment/kind/scripts/deploy.sh +++ b/deployment/kind/scripts/deploy.sh @@ -34,8 +34,8 @@ if [ -z `printenv S3_SECRET_KEY` ]; then echo "Stopping because S3_SECRET_KEY is undefined" exit 1 fi -if [ -z `printenv S3_ENDPOINT` ]; then - echo "Stopping because S3_ENDPOINT is undefined" +if [ -z `printenv S3_ENDPOINT_REGION` ]; then + echo "Stopping because S3_ENDPOINT_REGION is undefined" exit 1 fi if [ -z `printenv IONOS_TOKEN` ]; then @@ -72,7 +72,7 @@ helm install -n edc-ionos-s3 --wait vault hashicorp/vault \ export TF_VAR_kubeconfig=$KUBECONFIG export TF_VAR_s3_access_key=$S3_ACCESS_KEY export TF_VAR_s3_secret_key=$S3_SECRET_KEY -export TF_VAR_s3_endpoint=$S3_ENDPOINT +export TF_VAR_s3_endpoint_region=$S3_ENDPOINT_REGION export TF_VAR_ionos_token=$IONOS_TOKEN ../terraform/vault-init/vault-init.sh diff --git a/deployment/terraform/deploy-services.sh b/deployment/terraform/deploy-services.sh index 1d789f75..d533142e 100755 --- a/deployment/terraform/deploy-services.sh +++ b/deployment/terraform/deploy-services.sh @@ -19,8 +19,8 @@ if [[ -z `printenv TF_VAR_s3_secret_key` ]]; then exit 1 fi -if [[ -z `printenv TF_VAR_s3_endpoint` ]]; then - echo "Stopping because TF_VAR_s3_endpoint is undefined" +if [[ -z `printenv TF_VAR_s3_endpoint_region` ]]; then + echo "Stopping because TF_VAR_s3_endpoint_region is undefined" exit 1 fi diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf index 4d06a3d0..be8bd1d8 100644 --- a/deployment/terraform/ionos-s3-deploy/main.tf +++ b/deployment/terraform/ionos-s3-deploy/main.tf @@ -51,7 +51,7 @@ variable "pg_password" { default = "postgres" } -variable "s3_endpoint" {} +variable "s3_endpoint_region" {} variable "ionos_token" {} variable "vaultname" { @@ -86,8 +86,8 @@ resource "helm_release" "edc-ionos-s3" { } set { - name = "edc.ionos.endpoint" - value = var.s3_endpoint + name = "edc.ionos.endpoint.region" + value = var.s3_endpoint_region } set { diff --git a/deployment/terraform/vault-init/vault-init.sh b/deployment/terraform/vault-init/vault-init.sh index 4b87ca58..6931f968 100755 --- a/deployment/terraform/vault-init/vault-init.sh +++ b/deployment/terraform/vault-init/vault-init.sh @@ -41,7 +41,7 @@ kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR # Add secrets to Vault kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.access.key content=$TF_VAR_s3_access_key kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.secret.key content=$TF_VAR_s3_secret_key -kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint content=$TF_VAR_s3_endpoint +kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint.region content=$TF_VAR_s3_endpoint_region kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.token content=$TF_VAR_ionos_token kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.private.key content="$(cat ./certs/private.pem)" diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts index 400e9524..13d735e3 100644 --- a/extensions/core-ionos-s3/build.gradle.kts +++ b/extensions/core-ionos-s3/build.gradle.kts @@ -9,7 +9,6 @@ val metaModelVersion: String by project val minIOVersion: String by project val extensionsGroup: String by project val extensionsVersion: String by project -val junitVersion: String by project val gitHubPkgsName: String by project val gitHubPkgsUrl: String by project @@ -21,10 +20,6 @@ dependencies { implementation("${edcGroup}:transfer-spi:${edcVersion}") implementation("io.minio:minio:${minIOVersion}") - - testImplementation("${edcGroup}:junit:${edcVersion}") - testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") - testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") } java { @@ -32,10 +27,6 @@ java { withSourcesJar() } -tasks.test { - useJUnitPlatform() -} - publishing { publications { create("maven") { diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java similarity index 68% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java index be4140e0..38761d76 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java @@ -12,10 +12,10 @@ * */ -package com.ionos.edc.extension.s3.configuration; +package com.ionos.edc.extension.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3ConnectorApiImpl; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.connector.S3ConnectorImpl; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provides; @@ -25,13 +25,13 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_ACCESS_KEY; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_REGION; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_SECRET_KEY; -import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_ENDPOINT; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_TOKEN; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; -@Provides(S3ConnectorApi.class) +@Provides(S3Connector.class) @Extension(value = S3CoreExtension.NAME) public class S3CoreExtension implements ServiceExtension { @@ -53,20 +53,20 @@ public void initialize(ServiceExtensionContext context) { var accessKey = vault.resolveSecret(IONOS_ACCESS_KEY); var secretKey = vault.resolveSecret(IONOS_SECRET_KEY); - var endPoint = vault.resolveSecret(IONOS_ENDPOINT); + var region = vault.resolveSecret(IONOS_REGION); var token = vault.resolveSecret(IONOS_TOKEN); - if(accessKey == null || secretKey == null || endPoint ==null) { + if(accessKey == null || secretKey == null || region ==null || token == null) { monitor.warning("Couldn't connect or the vault didn't return values, falling back to ConfigMap Configuration"); - accessKey = context.getSetting(IONOS_ACCESS_KEY, IONOS_ACCESS_KEY); - secretKey = context.getSetting(IONOS_SECRET_KEY, IONOS_SECRET_KEY); - endPoint = context.getSetting(IONOS_ENDPOINT, IONOS_ENDPOINT); - token = context.getSetting(IONOS_TOKEN, IONOS_TOKEN); + accessKey = context.getSetting(IONOS_ACCESS_KEY, null); + secretKey = context.getSetting(IONOS_SECRET_KEY, null); + region = context.getSetting(IONOS_REGION, null); + token = context.getSetting(IONOS_TOKEN, null); } - var maxFiles = context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT); + var maxFiles = Integer.valueOf(context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT)); - var s3Api = new S3ConnectorApiImpl(endPoint, accessKey, secretKey, token, maxFiles); - context.registerService(S3ConnectorApi.class, s3Api); + var s3Connector = new S3ConnectorImpl(region, accessKey, secretKey, token, maxFiles); + context.registerService(S3Connector.class, s3Connector); } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java similarity index 65% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java index 738e2872..28955004 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java @@ -1,4 +1,18 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; +/* + * Copyright (c) 2024 IONOS + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * IONOS + * + */ + +package com.ionos.edc.extension.s3.api; public class S3AccessKey { public static final String AVAILABLE_STATUS = "AVAILABLE"; diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java new file mode 100644 index 00000000..5f3af422 --- /dev/null +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2024 IONOS + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * IONOS + * + */ + +package com.ionos.edc.extension.s3.api; + +import java.io.IOException; +import java.util.List; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.eclipse.edc.spi.EdcException; + +public class S3ApiClient { + private static final String BASE_URL = "https://s3.ionos.com"; + private static final String REGIONS_ENDPOINT_URL = BASE_URL + "/regions"; + private static final String ACCESS_KEYS_ENDPOINT_URL = BASE_URL + "/accesskeys"; + + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String BEARER_TOKEN_PREFIX = "Bearer "; + private static final String JSON_MEDIA_TYPE = "application/json"; + + private final OkHttpClient client; + private final ObjectMapper objectMapper; + + public S3ApiClient() { + client = new OkHttpClient(); + objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public List retrieveRegions(String token) { + + Request request = new Request.Builder().url(REGIONS_ENDPOINT_URL) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) + .get() + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new EdcException("Unexpected code [" + response.code() + "] retrieving S3 regions"); + } + + if (response.body() == null) + throw new IOException("Empty response body retrieving S3 regions"); + else + return objectMapper.readValue(response.body().string(), new TypeReference>() {}); + + } catch (IOException e) { + throw new EdcException("Error retrieving S3 accesskey", e); + } + } + + public S3AccessKey createAccessKey(String token) { + + Request request = new Request.Builder().url(ACCESS_KEYS_ENDPOINT_URL) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) + .post(RequestBody.create(MediaType.get(JSON_MEDIA_TYPE), new byte[0])) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new EdcException("Unexpected code [" + response.code() + "] creating S3 accesskey"); + } + + if (response.body() == null) + throw new IOException("Empty response body creating S3 accesskey"); + else + return objectMapper.readValue(response.body().string(), S3AccessKey.class); + + } catch (IOException e) { + throw new EdcException("Error creating S3 accesskey", e); + } + } + + public S3AccessKey retrieveAccessKey(String token, String keyID) { + String url = ACCESS_KEYS_ENDPOINT_URL + "/" + keyID; + + Request request = new Request.Builder().url(url) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) + .get() + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new EdcException("Unexpected code [" + response.code() + "] retrieving S3 accesskey"); + } + + if (response.body() == null) + throw new IOException("Empty response body retrieving S3 accesskey"); + else + return objectMapper.readValue(response.body().string(), S3AccessKey.class); + + } catch (IOException e) { + throw new EdcException("Error retrieving S3 accesskey", e); + } + } + + public void deleteAccessKey(String token, String keyID) { + String url = ACCESS_KEYS_ENDPOINT_URL + "/" + keyID; + + Request request = new Request.Builder().url(url) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) + .delete() + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new EdcException("Unexpected code [" + response.code() + "] deleting S3 accesskey"); + } + } catch (IOException e) { + throw new EdcException("Error deleting S3 accesskey", e); + } + } +} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java new file mode 100644 index 00000000..17d5b109 --- /dev/null +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 IONOS + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * IONOS + * + */ + +package com.ionos.edc.extension.s3.api; + +public class S3Region { + + private String id; + private Properties properties; + + public String getId() { + return id; + } + public Properties getProperties() { + return properties; + } + + public static class Properties { + private String endpoint; + + public String getEndpoint() { + return endpoint; + } + } +} \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java deleted file mode 100644 index 9413feed..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022 IONOS - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * IONOS - * - */ - -package com.ionos.edc.extension.s3.connector; - -import io.minio.MinioClient; - -public class MinioConnector { - - public MinioClient connect(String endpoint, String accessKey, String secretKey) { - return MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); - } -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java similarity index 80% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java index 9b7d3674..495e1c30 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java @@ -12,16 +12,17 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.connector; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3AccessKey; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; import java.io.ByteArrayInputStream; import java.util.List; @ExtensionPoint -public interface S3ConnectorApi { +public interface S3Connector { void createBucket(String bucketName); @@ -41,4 +42,5 @@ public interface S3ConnectorApi { void deleteAccessKey(String keyID); + S3Connector clone(String region, String accessKey, String secretKey); } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java similarity index 67% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java index 4f45c7c7..f1f3df31 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java @@ -12,12 +12,13 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.connector; -import com.ionos.edc.extension.s3.connector.MinioConnector; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; -import com.ionos.edc.extension.s3.connector.ionosapi.S3ApiConnector; +import com.ionos.edc.extension.s3.api.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3ApiClient; +import com.ionos.edc.extension.s3.api.S3Region; +import com.ionos.edc.extension.s3.types.S3Object; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; @@ -25,32 +26,33 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import org.eclipse.edc.spi.EdcException; +import org.jetbrains.annotations.NotNull; import java.io.ByteArrayInputStream; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -public class S3ConnectorApiImpl implements S3ConnectorApi { +import static com.ionos.edc.extension.s3.schema.IonosBucketSchema.REGION_ID_DEFAULT; - MinioConnector miniConnector = new MinioConnector(); - S3ApiConnector ionoss3Api = new S3ApiConnector(); +public class S3ConnectorImpl implements S3Connector { + + private final S3ApiClient S3ApiClient = new S3ApiClient(); private final MinioClient minioClient; - private final String region; - private String token; - private final Integer maxFiles; - - public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, int maxFiles) { - this.minioClient = miniConnector.connect(endpoint, accessKey, secretKey); - this.region = getRegion(endpoint); - this.token = ""; - this.maxFiles = maxFiles; - } + private final String regionId; + private final String token; + private final int maxFiles; - public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, String token, int maxFiles) { - this(endpoint, accessKey, secretKey, maxFiles); + public S3ConnectorImpl(String regionId, @NotNull String accessKey, @NotNull String secretKey, @NotNull String token, int maxFiles) { this.token = token; + this.maxFiles = maxFiles; + + this.regionId = Objects.requireNonNullElse(regionId, REGION_ID_DEFAULT); + var endpoint = getEndpoint( this.regionId , token); + + this.minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); } @Override @@ -59,7 +61,7 @@ public void createBucket(String bucketName) { try { minioClient.makeBucket(MakeBucketArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .build()); } catch (Exception e) { throw new EdcException("Creating bucket: " + e.getMessage()); @@ -72,7 +74,7 @@ public boolean bucketExists(String bucketName) { try { return minioClient.bucketExists(BucketExistsArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .build()); } catch (Exception e) { throw new EdcException("Verifying if bucket exists - " + e.getMessage()); @@ -88,7 +90,7 @@ public void uploadObject(String bucketName, String objectName, ByteArrayInputStr try { minioClient.putObject(PutObjectArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .object(objectName) .stream(stream, stream.available(), -1) .build()); @@ -105,7 +107,7 @@ public ByteArrayInputStream getObject(String bucketName, String objectName) { var request = GetObjectArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .object(objectName) .build(); @@ -124,7 +126,7 @@ public ByteArrayInputStream getObject(String bucketName, String objectName, long var request = GetObjectArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .object(objectName) .offset(offset) .length(length) @@ -142,7 +144,7 @@ public List listObjects(String bucketName, String objectName) { var objects = minioClient.listObjects(ListObjectsArgs.builder() .bucket(bucketName.toLowerCase()) - .region(region) + .region(regionId) .prefix(objectName) .recursive(true) .maxKeys(maxFiles) @@ -163,48 +165,43 @@ public List listObjects(String bucketName, String objectName) { @Override public S3AccessKey createAccessKey() { try{ - return ionoss3Api.createAccessKey(token); + return S3ApiClient.createAccessKey(token); } catch (Exception e) { - throw new EdcException("Creating temporary key - (Warning: max 5 keys on the storage) - " + e.getMessage()); + throw new EdcException("Error creating access key", e); } } @Override public S3AccessKey retrieveAccessKey(String keyID) { try{ - return ionoss3Api.retrieveAccessKey(token, keyID); + return S3ApiClient.retrieveAccessKey(token, keyID); } catch (Exception e) { - throw new EdcException("Retrieving temporary key: " + e.getMessage()); + throw new EdcException("Error retrieving access key", e); } } @Override public void deleteAccessKey(String keyID) { try{ - ionoss3Api.deleteAccessKey(token, keyID); + S3ApiClient.deleteAccessKey(token, keyID); } catch (Exception e) { - throw new EdcException("Deleting temporary key: " + e.getMessage()); + throw new EdcException("Error deleting access key", e); } } - static String getRegion(String endpoint) { - - switch (endpoint) { - case "https://s3-eu-central-1.ionoscloud.com": - return "de"; - case "s3-eu-central-1.ionoscloud.com": - return "de"; - case "https://s3-eu-central-2.ionoscloud.com": - return "eu-central-2"; - case "s3-eu-central-2.ionoscloud.com": - return "eu-central-2"; - case "https://s3-eu-south-2.ionoscloud.com": - return "eu-south-2"; - case "s3-eu-south-2.ionoscloud.com": - return "eu-south-2"; - default: - throw new EdcException("Invalid endpoint: " + endpoint); + private String getEndpoint(String regionId, String token) { + var regions = S3ApiClient.retrieveRegions(token); + + for (S3Region region: regions) { + if (region.getId().equals(regionId)) { + return "https://" + region.getProperties().getEndpoint(); + } } + throw new EdcException("Invalid region: " + regionId); } + @Override + public S3Connector clone(String region, String accessKey, String secretKey) { + return new S3ConnectorImpl(region, accessKey, secretKey, this.token, this.maxFiles); + } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java deleted file mode 100644 index d5bfc2e4..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; - -import java.io.IOException; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -import okhttp3.*; -import org.eclipse.edc.spi.EdcException; - -public class S3ApiConnector { - private static final String BASE_URL = "https://s3.ionos.com"; - - private final OkHttpClient client; - private final ObjectMapper objectMapper; - - public S3ApiConnector() { - client = new OkHttpClient(); - objectMapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - public S3AccessKey createAccessKey(String token) { - String url = BASE_URL + "/accesskeys"; - - Request request = new Request.Builder().url(url) - .addHeader("Authorization", "Bearer " + token) - .post(RequestBody.create(MediaType.get("application/json"), new byte[0])) - .build(); - - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] creating S3 accesskey"); - } - - if (response.body() == null) - throw new IOException("Empty response body creating S3 accesskey"); - else - return objectMapper.readValue(response.body().string(), S3AccessKey.class); - - } catch (IOException e) { - throw new EdcException("Error creating S3 accesskey", e); - } - } - - public S3AccessKey retrieveAccessKey(String token, String keyID) { - String url = BASE_URL + "/accesskeys/" + keyID; - - Request request = new Request.Builder().url(url) - .addHeader("Authorization", "Bearer " + token) - .get() - .build(); - - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] retrieving S3 accesskey"); - } - - if (response.body() == null) - throw new IOException("Empty response body retrieving S3 accesskey"); - else - return objectMapper.readValue(response.body().string(), S3AccessKey.class); - - } catch (IOException e) { - throw new EdcException("Error retrieving S3 accesskey", e); - } - } - - public void deleteAccessKey(String token, String keyID) { - String url = BASE_URL + "/accesskeys/" + keyID; - - Request request = new Request.Builder().url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) - .delete() - .build(); - - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] deleting S3 accesskey"); - } - } catch (IOException e) { - throw new EdcException("Error deleting S3 accesskey", e); - } - } -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java index 4f783c2a..740b3dd9 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java @@ -20,7 +20,7 @@ public interface IonosBucketSchema { String TYPE = "IonosS3"; - String STORAGE_NAME = EDC_NAMESPACE + "storage"; + String REGION_ID = EDC_NAMESPACE + "region"; String BUCKET_NAME = EDC_NAMESPACE + "bucketName"; String BLOB_NAME = EDC_NAMESPACE + "blobName"; String PATH = EDC_NAMESPACE + "path"; @@ -31,5 +31,5 @@ public interface IonosBucketSchema { String PUSH_TRANSFER_TYPE = TYPE + "-" + FlowType.PUSH; - String STORAGE_NAME_DEFAULT = "https://s3-eu-central-1.ionoscloud.com"; + String REGION_ID_DEFAULT = "de"; } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java index 456ed58a..3bef6158 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java @@ -17,7 +17,7 @@ public interface IonosSettingsSchema { String IONOS_ACCESS_KEY = "edc.ionos.access.key"; String IONOS_SECRET_KEY = "edc.ionos.secret.key"; - String IONOS_ENDPOINT = "edc.ionos.endpoint"; + String IONOS_REGION = "edc.ionos.endpoint.region"; String IONOS_TOKEN = "edc.ionos.token"; String IONOS_KEY_VALIDATION_ATTEMPTS = "edc.ionos.key.validation.attempts"; String IONOS_KEY_VALIDATION_DELAY = "edc.ionos.key.validation.delay"; diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java similarity index 95% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java index dec4b030..1ee51bf3 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java @@ -12,7 +12,7 @@ * */ -package com.ionos.edc.extension.s3.configuration; +package com.ionos.edc.extension.s3.types; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java similarity index 95% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java index 16bf41fd..7cd19e70 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java @@ -12,7 +12,7 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.types; public record S3Object(String objectName, long size) { diff --git a/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 7ac275e9..a9890db8 100644 --- a/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -12,5 +12,5 @@ # # -com.ionos.edc.extension.s3.configuration.S3CoreExtension +com.ionos.edc.extension.s3.S3CoreExtension diff --git a/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java b/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java deleted file mode 100644 index 2b48ff35..00000000 --- a/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.ionos.edc.extension.s3.api; - -import org.eclipse.edc.spi.EdcException; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class S3ConnectorApiImplTest { - - @Test - public void getRegion_Frankfurt_Full() { - var endpoint = "https://s3-eu-central-1.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("de", region); - } - - @Test - public void getRegion_Frankfurt_Short() { - var endpoint = "s3-eu-central-1.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("de", region); - } - - @Test - public void getRegion_Berlin_Full() { - var endpoint = "https://s3-eu-central-2.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("eu-central-2", region); - } - - @Test - public void getRegion_Berlin_Short() { - var endpoint = "s3-eu-central-2.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("eu-central-2", region); - } - - @Test - public void getRegion_Logrono_Full() { - var endpoint = "https://s3-eu-south-2.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("eu-south-2", region); - } - - @Test - public void getRegion_Logrono_Short() { - var endpoint = "s3-eu-south-2.ionoscloud.com"; - var region = S3ConnectorApiImpl.getRegion(endpoint); - - assertEquals("eu-south-2", region); - } - - @Test - public void getRegion_Invalid_Full() { - var endpoint = "https://s3-de-central.profitbricks.com"; - - Exception exception = assertThrows(EdcException.class, () -> { - S3ConnectorApiImpl.getRegion(endpoint); - }); - - assertNotNull(exception); - } - - public void getRegion_Invalid_Short() { - var endpoint = "s3-de-central.profitbricks.com"; - - Exception exception = assertThrows(EdcException.class, () -> { - S3ConnectorApiImpl.getRegion(endpoint); - }); - - assertNotNull(exception); - } -} diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java index a8d2cfcf..17b6c9b0 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java @@ -14,7 +14,7 @@ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer; import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; import org.eclipse.edc.runtime.metamodel.annotation.Extension; @@ -24,9 +24,6 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; -import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES; -import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; - @Extension(value = DataPlaneIonosS3Extension.NAME) public class DataPlaneIonosS3Extension implements ServiceExtension { @@ -35,7 +32,7 @@ public class DataPlaneIonosS3Extension implements ServiceExtension { private PipelineService pipelineService; @Inject - private S3ConnectorApi s3Api; + private S3Connector s3Connector; @Inject private DataTransferExecutorServiceContainer executorContainer; @@ -55,12 +52,10 @@ public String name() { public void initialize(ServiceExtensionContext context) { var monitor = context.getMonitor(); - var maxFiles = context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT); - - var sourceFactory = new IonosDataSourceFactory(s3Api, monitor); + var sourceFactory = new IonosDataSourceFactory(s3Connector, monitor); pipelineService.registerFactory(sourceFactory); - var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager, maxFiles); + var sinkFactory = new IonosDataSinkFactory(s3Connector, executorContainer.getExecutorService(), monitor, vault, typeManager); pipelineService.registerFactory(sinkFactory); context.getMonitor().info("File Transfer Extension initialized!"); } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java index 259645ca..edc1a547 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java @@ -14,7 +14,7 @@ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; @@ -30,7 +30,7 @@ public class IonosDataSink extends ParallelSink { - private S3ConnectorApi s3Api; + private S3Connector s3Connector; private String bucketName; private String path; @@ -81,7 +81,7 @@ protected StreamResult transferParts(List parts) { var byteArray = streamsOutput.toByteArray(); try (var streamsInput = new ByteArrayInputStream(byteArray)) { - s3Api.uploadObject(bucketName, blobName, streamsInput); + s3Connector.uploadObject(bucketName, blobName, streamsInput); streamsOutput.close(); } catch (Exception e) { @@ -122,8 +122,8 @@ public static Builder newInstance() { return new Builder(); } - public Builder s3Api(S3ConnectorApi s3Api) { - sink.s3Api = s3Api; + public Builder s3Connector(S3Connector s3Connector) { + sink.s3Connector = s3Connector; return this; } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java index a1a59288..b9d6dd38 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java @@ -15,9 +15,8 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.validation.IonosSinkDataAddressValidationRule; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3ConnectorApiImpl; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; @@ -34,26 +33,22 @@ import java.util.concurrent.ExecutorService; -import static com.ionos.edc.extension.s3.schema.IonosBucketSchema.STORAGE_NAME_DEFAULT; - public class IonosDataSinkFactory implements DataSinkFactory { private final ExecutorService executorService; private final Monitor monitor; - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Vault vault; private final TypeManager typeManager; - private final int maxFiles; private final Validator validator = new IonosSinkDataAddressValidationRule(); - public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager, int maxFiles) { - this.s3Api = s3Api; + public IonosDataSinkFactory(S3Connector s3Connector, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager) { + this.s3Connector = s3Connector; this.executorService = executorService; this.monitor = monitor; this.vault = vault; this.typeManager = typeManager; - this.maxFiles = maxFiles; } @Override @@ -77,45 +72,23 @@ public DataSink createSink(DataFlowStartMessage request) { var destination = request.getDestinationDataAddress(); - var secret = vault.resolveSecret(destination.getKeyName()); - if (secret != null) { - var token = typeManager.readValue(secret, IonosToken.class); - - if (destination.getStringProperty(IonosBucketSchema.STORAGE_NAME) != null) { - var s3ApiTemp = new S3ConnectorApiImpl(destination.getStringProperty(IonosBucketSchema.STORAGE_NAME), - token.getAccessKey(), - token.getSecretKey(), - maxFiles); - return IonosDataSink.Builder.newInstance() - .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) - .path(destination.getStringProperty(IonosBucketSchema.PATH)) - .requestId(request.getId()) - .executorService(executorService) - .monitor(monitor) - .s3Api(s3ApiTemp) - .build(); - } else { - var s3ApiTemp = new S3ConnectorApiImpl(STORAGE_NAME_DEFAULT, - token.getAccessKey(), - token.getSecretKey(), - maxFiles); - return IonosDataSink.Builder.newInstance() - .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) - .path(destination.getStringProperty(IonosBucketSchema.PATH)) - .requestId(request.getId()) - .executorService(executorService) - .monitor(monitor) - .s3Api(s3ApiTemp) - .build(); - } + var secret = vault.resolveSecret(request.getDestinationDataAddress().getKeyName()); + if (secret == null) { + throw new EdcException("Missing destination temporary token"); } + var token = typeManager.readValue(secret, IonosToken.class); + + var region = destination.getStringProperty(IonosBucketSchema.REGION_ID); + + var s3ConnectorTemp = s3Connector.clone(region, token.getAccessKey(), token.getSecretKey()); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .path(destination.getStringProperty(IonosBucketSchema.PATH)) - .requestId(request.getId()).executorService(executorService) + .requestId(request.getId()) + .executorService(executorService) .monitor(monitor) - .s3Api(s3Api) + .s3Connector(s3ConnectorTemp) .build(); } } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java index 7a7bd2b8..9f683b94 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java @@ -15,8 +15,8 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3Object; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; @@ -36,7 +36,7 @@ class IonosDataSource implements DataSource { - private S3ConnectorApi s3Api; + private S3Connector s3Connector; private Monitor monitor; private String bucketName; private String blobName; @@ -49,7 +49,7 @@ private IonosDataSource() { @Override public StreamResult> openPartStream() { - var objects = s3Api.listObjects(this.bucketName, this.blobName); + var objects = s3Connector.listObjects(this.bucketName, this.blobName); if (objects.isEmpty()) { return failure(new StreamFailure( @@ -76,7 +76,7 @@ public StreamResult> openPartStream() { } List parts = objects.stream() - .map(object -> new S3Part(s3Api, monitor, bucketName, object.objectName(), object.isDirectory(), object.size())) + .map(object -> new S3Part(s3Connector, monitor, bucketName, object.objectName(), object.isDirectory(), object.size())) .collect(Collectors.toList()); return success(parts.stream()); } @@ -99,7 +99,8 @@ public void close() { } public static class S3Part implements Part { - private final S3ConnectorApi s3Api; + + private final S3Connector s3Connector; private final Monitor monitor; private final String bucketName; private final String blobName; @@ -109,9 +110,9 @@ public static class S3Part implements Part { private boolean isOpened = true; private long currentOffset = 0; - S3Part(S3ConnectorApi s3Api, Monitor monitor, String bucketName, String blobName, boolean isDirectory, long fileSize) { + S3Part(S3Connector s3Connector, Monitor monitor, String bucketName, String blobName, boolean isDirectory, long fileSize) { super(); - this.s3Api = s3Api; + this.s3Connector = s3Connector; this.monitor = monitor; this.bucketName = bucketName; this.blobName = blobName; @@ -141,9 +142,9 @@ public InputStream openStream() { InputStream stream; if (isDirectory || (fileSize <= chunkSize())) { - stream = s3Api.getObject(bucketName, blobName); + stream = s3Connector.getObject(bucketName, blobName); } else { - stream = s3Api.getObject(bucketName, blobName, currentOffset, chunkSize()); + stream = s3Connector.getObject(bucketName, blobName, currentOffset, chunkSize()); } if (!isDirectory) { @@ -174,8 +175,8 @@ public static Builder newInstance() { return new Builder(); } - public Builder client(S3ConnectorApi s3Api) { - source.s3Api = s3Api; + public Builder client(S3Connector s3Connector) { + source.s3Connector = s3Connector; return this; } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java index 2b2e4b80..13c95c68 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java @@ -15,7 +15,7 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.validation.IonosSourceDataAddressValidationRule; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; @@ -29,13 +29,13 @@ import org.jetbrains.annotations.NotNull; public class IonosDataSourceFactory implements DataSourceFactory { - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Monitor monitor; private final Validator validator = new IonosSourceDataAddressValidationRule(); - public IonosDataSourceFactory(S3ConnectorApi s3Api, Monitor monitor) { - this.s3Api = s3Api; + public IonosDataSourceFactory(S3Connector s3Connector, Monitor monitor) { + this.s3Connector = s3Connector; this.monitor = monitor; } @@ -61,7 +61,7 @@ public DataSource createSource(DataFlowStartMessage request) { var source = request.getSourceDataAddress(); return IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(source.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .blobName(source.getStringProperty(IonosBucketSchema.BLOB_NAME)) diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/validation/IonosSourceDataAddressValidationRule.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/validation/IonosSourceDataAddressValidationRule.java index c9691b10..02e7042a 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/validation/IonosSourceDataAddressValidationRule.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/validation/IonosSourceDataAddressValidationRule.java @@ -28,7 +28,7 @@ public class IonosSourceDataAddressValidationRule implements Validator { var value = dataAddress.getStringProperty(it); if (value == null || value.isBlank()) { diff --git a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java index df97cfb8..149e5676 100644 --- a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java +++ b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java @@ -3,8 +3,8 @@ */ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3Object; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; import org.eclipse.edc.spi.monitor.Monitor; import org.junit.jupiter.api.BeforeEach; @@ -37,7 +37,7 @@ class IonosDataSourceTest { private static final String TEST_SUB_FOLDER_2_NAME = "device2/"; @Mock - private S3ConnectorApi s3Api; + private S3Connector s3Connector; @Mock private Monitor monitor; @@ -50,10 +50,10 @@ public void setup() { public void openPartStream_empty() { doReturn(List.of()) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FILE_1_NAME) @@ -70,10 +70,10 @@ public void openPartStream_singleFile() { var s3Objects = List.of(new S3Object(TEST_FILE_1_NAME, TEST_FILE_1_SIZE)); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FILE_1_NAME) @@ -99,10 +99,10 @@ public void openPartStream_folder() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -140,10 +140,10 @@ public void openPartStream_folder_includeFiles() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -178,10 +178,10 @@ public void openPartStream_folder_excludeFiles() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -213,10 +213,10 @@ public void openPartStream_subFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -259,10 +259,10 @@ public void openPartStream_subFolder_includeFilesInFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -298,10 +298,10 @@ public void openPartStream_subFolder_excludeFilesInFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -342,10 +342,10 @@ public void openPartStream_subFolder_includeFilesInSubFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -381,10 +381,10 @@ public void openPartStream_subFolder_excludeFilesInSubFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java index 2e091473..d066b207 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java @@ -14,8 +14,8 @@ package com.ionos.edc.provision.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; import com.ionos.edc.provision.s3.bucket.IonosS3ConsumerResourceDefinitionGenerator; import com.ionos.edc.provision.s3.bucket.IonosS3ProvisionedResource; import com.ionos.edc.provision.s3.bucket.IonosS3Provisioner; @@ -45,7 +45,7 @@ public class IonosProvisionExtension implements ServiceExtension { @Inject private TypeManager typeManager; @Inject - private S3ConnectorApi clientApi; + private S3Connector clientApi; @Override public String name() { diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java index abc01231..4f86d32d 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java @@ -44,7 +44,7 @@ public class IonosS3ConsumerResourceDefinitionGenerator implements ConsumerResou var id = randomUUID().toString(); var keyName = destination.getKeyName(); - var storage = destination.getStringProperty(IonosBucketSchema.STORAGE_NAME); + var regionId = destination.getStringProperty(IonosBucketSchema.REGION_ID); var bucketName = destination.getStringProperty(IonosBucketSchema.BUCKET_NAME); var accessKey = destination.getStringProperty(IonosBucketSchema.ACCESS_KEY_ID); var secretKey = destination.getStringProperty(IonosBucketSchema.SECRET_ACCESS_KEY); @@ -52,7 +52,7 @@ public class IonosS3ConsumerResourceDefinitionGenerator implements ConsumerResou return IonosS3ResourceDefinition.Builder.newInstance() .id(id) .keyName(keyName) - .storage(storage) + .regionId(regionId) .bucketName(bucketName) .path(path) .accessKey(accessKey) diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java index 3a768d88..b48f0fb1 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java @@ -50,8 +50,8 @@ public static Builder newInstance() { return new Builder(); } - public Builder storage(String storage) { - dataAddressBuilder.property(STORAGE_NAME, storage); + public Builder regionId(String regionId) { + dataAddressBuilder.property(REGION_ID, regionId); return this; } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java index 2417c18b..e2149d29 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java @@ -14,10 +14,10 @@ package com.ionos.edc.provision.s3.bucket; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3AccessKey; import dev.failsafe.RetryPolicy; import org.eclipse.edc.connector.controlplane.transfer.spi.provision.Provisioner; import org.eclipse.edc.connector.controlplane.transfer.spi.types.DeprovisionedResource; @@ -37,15 +37,15 @@ public class IonosS3Provisioner implements Provisioner retryPolicy; - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Integer keyValidationAttempts; private final Long keyValidationDelay; - public IonosS3Provisioner(Monitor monitor, RetryPolicy retryPolicy, S3ConnectorApi s3Api, int keyValidationAttempts, long keyValidationDelay) { + public IonosS3Provisioner(Monitor monitor, RetryPolicy retryPolicy, S3Connector s3Connector, int keyValidationAttempts, long keyValidationDelay) { this.monitor = monitor; this.retryPolicy = retryPolicy; - this.s3Api = s3Api; + this.s3Connector = s3Connector; this.keyValidationAttempts = keyValidationAttempts; this.keyValidationDelay = keyValidationDelay; } @@ -64,7 +64,7 @@ public boolean canDeprovision(ProvisionedResource resourceDefinition) { public CompletableFuture> provision(IonosS3ResourceDefinition resourceDefinition, Policy policy) { String bucketName = resourceDefinition.getBucketName(); - if (!s3Api.bucketExists(bucketName)) { + if (!s3Connector.bucketExists(bucketName)) { createBucket(bucketName); } @@ -80,8 +80,8 @@ public CompletableFuture> provision(IonosS3Resou .accessKeyID(temporaryKey.getId()) .transferProcessId(resourceDefinition.getTransferProcessId()) .hasToken(true); - if (resourceDefinition.getStorage() != null) { - resourceBuilder = resourceBuilder.storage(resourceDefinition.getStorage()); + if (resourceDefinition.getRegionId() != null) { + resourceBuilder = resourceBuilder.regionId(resourceDefinition.getRegionId()); } if (resourceDefinition.getPath() != null) { resourceBuilder = resourceBuilder.path(resourceDefinition.getPath()); @@ -99,14 +99,14 @@ public CompletableFuture> provision(IonosS3Resou @Override public CompletableFuture> deprovision(IonosS3ProvisionedResource provisionedResource, Policy policy) { - return with(retryPolicy).runAsync(() -> s3Api.deleteAccessKey(provisionedResource.getAccessKeyID())) + return with(retryPolicy).runAsync(() -> s3Connector.deleteAccessKey(provisionedResource.getAccessKeyID())) .thenApply(empty -> StatusResult.success(DeprovisionedResource.Builder.newInstance().provisionedResourceId(provisionedResource.getId()).build()) ); } private S3AccessKey createTemporaryKey() { - var accessKey = s3Api.createAccessKey(); + var accessKey = s3Connector.createAccessKey(); // Validate the temporary key var validated = false; @@ -124,7 +124,7 @@ private S3AccessKey createTemporaryKey() { return accessKey; } else { // Delete the not validated temporary key - s3Api.deleteAccessKey(accessKey.getId()); + s3Connector.deleteAccessKey(accessKey.getId()); throw new EdcException("Temporary key not validated after " + attempts + " attempts of " + keyValidationDelay + " ms"); } } @@ -138,12 +138,12 @@ private boolean validateKey(S3AccessKey accessKey) { } // Validate the key status - var retrievedAccessKey = s3Api.retrieveAccessKey(accessKey.getId()); + var retrievedAccessKey = s3Connector.retrieveAccessKey(accessKey.getId()); return (retrievedAccessKey.getMetadata().getStatus().equals(S3AccessKey.AVAILABLE_STATUS)); } private void createBucket(String bucketName) { - s3Api.createBucket(bucketName); + s3Connector.createBucket(bucketName); } } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java index a0c9decb..10b7b8cf 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java @@ -24,7 +24,7 @@ public class IonosS3ResourceDefinition extends ResourceDefinition { private String keyName; - private String storage; + private String regionId; private String bucketName; private String path; private String accessKey; @@ -36,8 +36,8 @@ public IonosS3ResourceDefinition() { public String getKeyName() { return keyName; } - public String getStorage() { - return storage; + public String getRegionId() { + return regionId; } public String getBucketName() { return bucketName; @@ -50,7 +50,7 @@ public String getPath() { public Builder toBuilder() { return initializeBuilder(new Builder()) .keyName(keyName) - .storage(storage) + .regionId(regionId) .bucketName(bucketName) .path(path) .accessKey(accessKey) @@ -71,8 +71,8 @@ public Builder keyName(String keyName) { resourceDefinition.keyName = keyName; return this; } - public Builder storage(String storage) { - resourceDefinition.storage = storage; + public Builder regionId(String regionId) { + resourceDefinition.regionId = regionId; return this; } public Builder bucketName(String bucketName) { diff --git a/hashicorp/README.md b/hashicorp/README.md index d2415bb1..d6a12a8f 100644 --- a/hashicorp/README.md +++ b/hashicorp/README.md @@ -64,7 +64,7 @@ vault login token= vault kv put secret/edc.ionos.access.key content= vault kv put secret/edc.ionos.secret.key content= vault kv put secret/edc.ionos.token content= -vault kv put secret/edc.ionos.endpoint content= +vault kv put secret/edc.ionos.endpoint.region content= ``` Note: @@ -76,7 +76,7 @@ Note: ```console kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.access.key content= kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.secret.key content= -kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.endpoint content= +kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content= kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.token content= ``` diff --git a/launchers/dev/connector-consumer/resources/config.properties b/launchers/dev/connector-consumer/resources/config.properties index 22261f56..eac360ef 100644 --- a/launchers/dev/connector-consumer/resources/config.properties +++ b/launchers/dev/connector-consumer/resources/config.properties @@ -23,5 +23,5 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= -edc.ionos.endpoint= -edc.ionos.token= +edc.ionos.endpoint.region= +edc.ionos.token= diff --git a/launchers/dev/connector-provider/resources/config.properties b/launchers/dev/connector-provider/resources/config.properties index c24f6764..9e5211c7 100644 --- a/launchers/dev/connector-provider/resources/config.properties +++ b/launchers/dev/connector-provider/resources/config.properties @@ -23,4 +23,5 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= -edc.ionos.endpoint= +edc.ionos.endpoint.region= +edc.ionos.token= diff --git a/launchers/prod/connector-persistence/resources/config.properties b/launchers/prod/connector-persistence/resources/config.properties index d65abebb..418b9d39 100644 --- a/launchers/prod/connector-persistence/resources/config.properties +++ b/launchers/prod/connector-persistence/resources/config.properties @@ -23,8 +23,8 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= -edc.ionos.endpoint= -edc.ionos.token= +edc.ionos.endpoint.region= +edc.ionos.token= edc.datasource.asset.name=asset edc.datasource.asset.user= diff --git a/launchers/prod/connector/resources/config.properties b/launchers/prod/connector/resources/config.properties index 6cb13c56..2b1aacd5 100644 --- a/launchers/prod/connector/resources/config.properties +++ b/launchers/prod/connector/resources/config.properties @@ -23,5 +23,5 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= -edc.ionos.endpoint= -edc.ionos.token= +edc.ionos.endpoint.region= +edc.ionos.token=