diff --git a/.gitignore b/.gitignore index 53ded747..825ff2c4 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ launchers/demo-e2e/edc-config.properties *.hprof **/vault-keys.json +**/vault-tokens.json runtime_settings.properties generated_backend.tf diff --git a/README.md b/README.md index 5bf2e2d0..472f32ac 100644 --- a/README.md +++ b/README.md @@ -59,15 +59,15 @@ The credentials can be found/configured in one of the following: It is required to configure those parameters: -| Parameter name | Description | Mandatory | -|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| -| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | -| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | -| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | -| `edc.ionos.endpoint.region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | No, the default value is de | -| `edc.ionos.max.files` | Maximum number of files retrieved by list files function. | No, the default value is 5,000 files | -| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation. | No, the default values is 10 attempts | -| `edc.ionos.key.validation.delay` | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number. | No, the default value is 3,000 (3 seconds) | +| Parameter name | Description | Mandatory | +|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| +| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | +| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | +| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | +| `edc.ionos.endpoint.region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | No, the default value is "de" | +| `edc.ionos.max.files` | Maximum number of files retrieved by list files function. | No, the default value is 5,000 files | +| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation. | No, the default values is 10 attempts | +| `edc.ionos.key.validation.delay` | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number. | No, the default value is 3,000 (3 seconds) | To create the token please take a look at the following [documentation](./ionos_token.md). diff --git a/assets.md b/assets.md index 69494c8f..9af0d989 100644 --- a/assets.md +++ b/assets.md @@ -11,11 +11,11 @@ The asset registration aims to specify which file/folder we want to share. We ca | Parameter | Description | Mandatory | |------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| -| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information. | no, default value = de | -| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes | -| `blobName` | File name or path to folder | yes | -| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer
* do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied | no | -| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer
| no | +| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information.| no, default value = de | +| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.| yes | +| `blobName` | File name or path to folder| yes | +| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer
* do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied| no | +| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer
| no | Note: if `filterIncludes` and `filterExcludes` parameters are satisfied, the files to be copied will be selected using the `filterIncludes` and after that selected list, the files that have the pattern defined in the `filterExcludes` will be ignored. @@ -23,9 +23,9 @@ Note: if `filterIncludes` and `filterExcludes` parameters are satisfied, the f ## Example ```json -"dataAddress": { +"dataAddress":{ "type": "IonosS3", //from EDC - "region": "de, + "region": "de", "bucketName": "mybucket", "blobName": "folder1/", "filterIncludes": "file1.csv", @@ -40,17 +40,17 @@ The transfer of assets aims to transfer the files/folders from one connector to ### Requirements -| Parameter | Description | Mandatory | -|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| -| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information. | no, default value = de | -| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes | -| `path` | Path of destination where the file/folder will be placed.
*if the path not filled, the file will be placed in the root of the bucket. | no | +| Parameter | Description | Mandatory | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| `region` | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints) for further information.| no, default value = de | +| `bucketName` | IONOS S3 bucket name. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information| yes | +| `path` | Path of destination where the file/folder will be placed.
*if the path not filled, the file will be placed in the root of the bucket. | no | ## Example ```json -"dataDestination": { +"dataDestination":{ "type": "IonosS3", //from EDC "region": "de", "bucketName": "mybucket", diff --git a/deployment/README.md b/deployment/README.md index d3d5ada6..8b6acd43 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -86,13 +86,13 @@ In case you want to configure this Connector without Hashicorp Vault, you need t ```yaml ionos: - region: + region: accessKey: secretKey: token: ``` -They should be the same as the ones set in the environment variables. The **ionos.endpoint** is set to the default S3 location, but it can be changed to any other location. +They should be the same as the ones set in the environment variables. The **ionos.region** is set to the default S3 endpoint region, but it can be changed to any other location. If you don't want the Connector to be externally accessible, you need to set the following parameters in the helm [values.yaml](deployment/helm/edc-ionos-s3/values.yaml): @@ -110,21 +110,19 @@ This will allocate a public IP address to the Connector. You can then access it All commands paths are relative to the current directory where this readme is located. -### 1. Install the EDC Ionos S3 services +### 1. Deploy the services -To install the services run the script ```deploy-services.sh``` in ```terraform``` directory. +To deploy the services run the script ```deploy-services.sh``` in ```terraform``` directory. ```sh cd terraform ./deploy-services.sh ``` +### 2. Undeploy the services -### 2. Vault keys -After the services are installed you will have ```vault-keys.json``` file containing the vault keys in ```terraform``` directory. - -### 3. Destroy the services +To undeploy the services run the script ```undeploy-services.sh``` in ```terraform``` directory. ```sh cd terraform -./destroy-services.sh +./undeploy-services.sh ``` diff --git a/deployment/helm/edc-ionos-s3/templates/configmap.yaml b/deployment/helm/edc-ionos-s3/templates/configmap.yaml index aa81e0af..e094843f 100644 --- a/deployment/helm/edc-ionos-s3/templates/configmap.yaml +++ b/deployment/helm/edc-ionos-s3/templates/configmap.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "edc-ionos-s3.fullname" . }}-config data: config.properties: | + edc.participant.id={{ .Values.edc.participant.id }} web.http.port={{ .Values.web.http.port }} web.http.path={{ .Values.web.http.path }} web.http.management.port={{ .Values.web.http.management.port }} @@ -14,20 +15,19 @@ data: web.http.public.path={{ .Values.web.http.public.path }} web.http.control.port={{ .Values.web.http.control.port }} web.http.control.path={{ .Values.web.http.control.path }} + edc.dsp.callback.address={{ .Values.edc.dsp.callback.address }} + edc.dataplane.token.validation.endpoint={{ .Values.edc.dataplane.token.validation.endpoint }} + edc.dataplane.api.public.baseurl={{ .Values.edc.dataplane.api.public.baseurl }} edc.api.auth.key={{ .Values.edc.api.auth.key }} - edc.participant.id={{ .Values.edc.participant.id }} - edc.ionos.access.key={{ .Values.edc.ionos.accessKey }} - edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }} - edc.ionos.endpoint.region={{ .Values.edc.ionos.endpoint.region }} - edc.ionos.token={{ .Values.edc.ionos.token }} + edc.transfer.proxy.token.signer.privatekey.alias={{ .Values.edc.vault.certificates.privateKey.alias }} + edc.transfer.proxy.token.verifier.publickey.alias={{ .Values.edc.vault.certificates.publicKey.alias }} edc.vault.hashicorp.url={{ .Values.edc.vault.hashicorp.url }} edc.vault.hashicorp.token={{ .Values.edc.vault.hashicorp.token }} edc.vault.hashicorp.timeout.seconds={{ .Values.edc.vault.hashicorp.timeout.seconds }} - edc.ids.id={{ .Values.edc.ids.id }} - edc.dsp.callback.address={{ .Values.edc.dsp.callback.address }}:{{ .Values.web.http.protocol.port }}{{ .Values.web.http.protocol.path }} - edc.receiver.http.endpoint={{ .Values.edc.receiver.http.endpoint }}/receiver/{{ .Values.edc.ids.id }}/callback - edc.public.key.alias={{ .Values.edc.public.key.alias }} - edc.dataplane.token.validation.endpoint={{ .Values.edc.dataplane.token.validation.endpoint }}:{{ .Values.web.http.control.port }}{{ .Values.web.http.control.path }}/token + edc.ionos.access.key={{ .Values.edc.ionos.accessKey }} + edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }} + edc.ionos.endpoint.region={{ .Values.edc.ionos.region }} + edc.ionos.token={{ .Values.edc.ionos.token }} {{- if eq .Values.edc.persistenceType "PostgreSQLaaS" }} edc.datasource.asset.name=asset diff --git a/deployment/helm/edc-ionos-s3/values.yaml b/deployment/helm/edc-ionos-s3/values.yaml index 0de447f3..d874e6ea 100644 --- a/deployment/helm/edc-ionos-s3/values.yaml +++ b/deployment/helm/edc-ionos-s3/values.yaml @@ -113,34 +113,31 @@ edc: auth: key: password vault: - clientid: company1 - tenantid: 1 - certificate: /resources/ + certificates: + publicKey: + alias: edc.connector.public.key + privateKey: + alias: edc.connector.private.key hashicorp: url: http://vault:8200 token: timeout: seconds: 30 - ids: - id: urn:connector:provider ionos: - endpoint: s3-eu-central-1.ionoscloud.com + region: de accessKey: notnull secretKey: notnull token: notnull dsp: callback: - address: http://localhost - receiver: - http: - endpoint: http://localhost:4000 - public: - key: - alias: alias + address: http://localhost:8281/protocol dataplane: + api: + public: + baseurl: http://localhost:8282/public token: validation: - endpoint: http://localhost + endpoint: http://localhost:8283/control/token persistenceType: PostgreSQLaaS # 'PostgreSQLaaS', 'PostgreSQL' or 'None' postgresql: # Only used if persistenceType is 'PostgreSQLaaS' or 'PostgreSQL' host: postgresql diff --git a/deployment/terraform/clean-state.sh b/deployment/terraform/clean-state.sh new file mode 100755 index 00000000..6d8d4a10 --- /dev/null +++ b/deployment/terraform/clean-state.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# remove terraform state +rm -rf ./configure-public-address/.terraform +rm -f ./configure-public-address/terraform.tfstate +rm -f ./configure-public-address/.terraform.lock.hcl +rm -f ./configure-public-address/terraform.tfstate.backup + +rm -rf ./ionos-s3-deploy/.terraform +rm -f ./ionos-s3-deploy/terraform.tfstate +rm -f ./ionos-s3-deploy/.terraform.lock.hcl +rm -f ./ionos-s3-deploy/terraform.tfstate.backup + +rm -rf ./vault-init/.terraform +rm -f ./vault-init/terraform.tfstate +rm -f ./vault-init/.terraform.lock.hcl +rm -f ./vault-init/terraform.tfstate.backup + +rm -rf ./vault-deploy/.terraform +rm -f ./vault-deploy/terraform.tfstate +rm -f ./vault-deploy/.terraform.lock.hcl +rm -f ./vault-deploy/terraform.tfstate.backup + +rm -rf ./ionos-postgresqlaas/.terraform +rm -f ./ionos-postgresqlaas/terraform.tfstate +rm -f ./ionos-postgresqlaas/.terraform.lock.hcl +rm -f ./ionos-postgresqlaas/terraform.tfstate.backup + +rm -rf ./postgresql-deploy/.terraform +rm -f ./postgresql-deploy/terraform.tfstate +rm -f ./postgresql-deploy/.terraform.lock.hcl +rm -f ./postgresql-deploy/terraform.tfstate.backup + +rm -rf ./db-scripts/.terraform +rm -f ./db-scripts/terraform.tfstate +rm -f ./db-scripts/.terraform.lock.hcl +rm -f ./db-scripts/terraform.tfstate.backup + +rm -f vault-init/vault-keys.json +rm -f vault-init/vault-tokens.json + +echo "Terraform state cleanup complete" \ No newline at end of file diff --git a/deployment/terraform/configure-public-address/public-addresses.sh b/deployment/terraform/configure-public-address/public-addresses.sh index d293ddc9..bbd41ecd 100755 --- a/deployment/terraform/configure-public-address/public-addresses.sh +++ b/deployment/terraform/configure-public-address/public-addresses.sh @@ -12,9 +12,9 @@ fi # Change public address in the config.properties in the configmap kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dsp.callback.address=.*/edc.dsp.callback.address=http:\/\/$CONNECTOR_ADDRESS:8281\/protocol/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f - -kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.receiver.http.endpoint=.*/edc.receiver.http.endpoint=http:\/\/$CONNECTOR_ADDRESS:4000\/receiver\/urn:connector:provider\/callback/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f - - kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dataplane.token.validation.endpoint=.*/edc.dataplane.token.validation.endpoint=http:\/\/$CONNECTOR_ADDRESS:8283\/control\/token/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f - +kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dataplane.api.public.baseurl=.*/edc.dataplane.api.public.baseurl=http:\/\/$CONNECTOR_ADDRESS:8282\/public/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f - + # Restart the pods kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace delete pod -l app.kubernetes.io/name=edc-ionos-s3 \ No newline at end of file diff --git a/deployment/terraform/db-scripts/accesstokendata-store/schema.sql b/deployment/terraform/db-scripts/accesstokendata-store/schema.sql new file mode 100644 index 00000000..de43bc74 --- /dev/null +++ b/deployment/terraform/db-scripts/accesstokendata-store/schema.sql @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_accesstokendata +( + id VARCHAR NOT NULL PRIMARY KEY, + claim_token JSON NOT NULL, + data_address JSON NOT NULL, + additional_properties JSON DEFAULT '{}' +); + +COMMENT ON COLUMN edc_accesstokendata.claim_token IS 'ClaimToken serialized as JSON map'; +COMMENT ON COLUMN edc_accesstokendata.data_address IS 'DataAddress serialized as JSON map'; +COMMENT ON COLUMN edc_accesstokendata.additional_properties IS 'Optional Additional properties serialized as JSON map'; diff --git a/deployment/terraform/db-scripts/asset-index/schema.sql b/deployment/terraform/db-scripts/asset-index/schema.sql new file mode 100644 index 00000000..6274b5f8 --- /dev/null +++ b/deployment/terraform/db-scripts/asset-index/schema.sql @@ -0,0 +1,30 @@ +-- +-- Copyright (c) 2022 - 2023 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - improvements +-- + +-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES + +-- table: edc_asset +CREATE TABLE IF NOT EXISTS edc_asset +( + asset_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + properties JSON DEFAULT '{}', + private_properties JSON DEFAULT '{}', + data_address JSON DEFAULT '{}', + PRIMARY KEY (asset_id) +); + +COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; diff --git a/deployment/terraform/db-scripts/contract-definition-store/schema.sql b/deployment/terraform/db-scripts/contract-definition-store/schema.sql new file mode 100644 index 00000000..98a30b7e --- /dev/null +++ b/deployment/terraform/db-scripts/contract-definition-store/schema.sql @@ -0,0 +1,27 @@ +-- +-- Copyright (c) 2022 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Microsoft Corporation - refactoring +-- SAP SE - add private properties to contract definition +-- + +-- table: edc_contract_definitions +-- only intended for and tested with H2 and Postgres! +CREATE TABLE IF NOT EXISTS edc_contract_definitions +( + created_at BIGINT NOT NULL, + contract_definition_id VARCHAR NOT NULL, + access_policy_id VARCHAR NOT NULL, + contract_policy_id VARCHAR NOT NULL, + assets_selector JSON NOT NULL, + private_properties JSON, + PRIMARY KEY (contract_definition_id) +); diff --git a/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql b/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql new file mode 100644 index 00000000..02d64c49 --- /dev/null +++ b/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql @@ -0,0 +1,86 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER DEFAULT 60000 NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); + + + +CREATE TABLE IF NOT EXISTS edc_contract_agreement +( + agr_id VARCHAR NOT NULL + CONSTRAINT contract_agreement_pk + PRIMARY KEY, + provider_agent_id VARCHAR, + consumer_agent_id VARCHAR, + signing_date BIGINT, + start_date BIGINT, + end_date INTEGER, + asset_id VARCHAR NOT NULL, + policy JSON +); + + +CREATE TABLE IF NOT EXISTS edc_contract_negotiation +( + id VARCHAR NOT NULL + CONSTRAINT contract_negotiation_pk + PRIMARY KEY, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + correlation_id VARCHAR, + counterparty_id VARCHAR NOT NULL, + counterparty_address VARCHAR NOT NULL, + protocol VARCHAR NOT NULL, + type VARCHAR NOT NULL, + state INTEGER DEFAULT 0 NOT NULL, + state_count INTEGER DEFAULT 0, + state_timestamp BIGINT, + error_detail VARCHAR, + agreement_id VARCHAR + CONSTRAINT contract_negotiation_contract_agreement_id_fk + REFERENCES edc_contract_agreement, + contract_offers JSON, + callback_addresses JSON, + trace_context JSON, + pending BOOLEAN DEFAULT FALSE, + protocol_messages JSON, + lease_id VARCHAR + CONSTRAINT contract_negotiation_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; + + +CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index + ON edc_contract_negotiation (correlation_id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex + ON edc_contract_negotiation (id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex + ON edc_contract_agreement (agr_id); + + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS contract_negotiation_state ON edc_contract_negotiation (state,state_timestamp); \ No newline at end of file diff --git a/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql b/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql new file mode 100644 index 00000000..0f2f3246 --- /dev/null +++ b/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + + +CREATE TABLE IF NOT EXISTS edc_data_plane_instance +( + id VARCHAR NOT NULL PRIMARY KEY, + data JSON, + lease_id VARCHAR + CONSTRAINT data_plane_instance_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); diff --git a/deployment/terraform/db-scripts/data-plane-store/schema.sql b/deployment/terraform/db-scripts/data-plane-store/schema.sql new file mode 100644 index 00000000..768320ac --- /dev/null +++ b/deployment/terraform/db-scripts/data-plane-store/schema.sql @@ -0,0 +1,43 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_data_plane +( + process_id VARCHAR NOT NULL PRIMARY KEY, + state INTEGER NOT NULL , + created_at BIGINT NOT NULL , + updated_at BIGINT NOT NULL , + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + trace_context JSON, + error_detail VARCHAR, + callback_address VARCHAR, + lease_id VARCHAR + CONSTRAINT data_plane_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL, + source JSON, + destination JSON, + properties JSON, + flow_type VARCHAR +); + +COMMENT ON COLUMN edc_data_plane.trace_context IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.source IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.destination IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.properties IS 'Java Map serialized as JSON'; + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS data_plane_state ON edc_data_plane (state,state_time_stamp); diff --git a/deployment/terraform/db-scripts/db-scripts.sh b/deployment/terraform/db-scripts/db-scripts.sh new file mode 100755 index 00000000..5dbdfe98 --- /dev/null +++ b/deployment/terraform/db-scripts/db-scripts.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +echo "Creating database $TF_VAR_pg_database" +set +e +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="postgres" --command="CREATE DATABASE $TF_VAR_pg_database;" +set -e + +echo "Creating accesstokendata-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-accesstokendata --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./accesstokendata-store/schema.sql + +echo "Creating asset-index tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-asset-index --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./asset-index/schema.sql + +echo "Creating contract-definition-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-definition-store/schema.sql + +echo "Creating contract-negotiation-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-negotiation --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-negotiation-store/schema.sql + +echo "Creating data-plane-instance-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane-instance --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-instance-store/schema.sql + +echo "Creating data-plane-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-store/schema.sql + +echo "Creating edr-index tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-edr --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./edr-index/schema.sql + +echo "Creating policy-definition-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-policy-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./policy-definition-store/schema.sql + +echo "Creating transfer-process-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-transfer-process --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./transfer-process-store/schema.sql diff --git a/deployment/terraform/db-scripts/edr-index/schema.sql b/deployment/terraform/db-scripts/edr-index/schema.sql new file mode 100644 index 00000000..3f305856 --- /dev/null +++ b/deployment/terraform/db-scripts/edr-index/schema.sql @@ -0,0 +1,11 @@ + +CREATE TABLE IF NOT EXISTS edc_edr_entry +( + transfer_process_id VARCHAR NOT NULL PRIMARY KEY, + agreement_id VARCHAR NOT NULL, + asset_id VARCHAR NOT NULL, + provider_id VARCHAR NOT NULL, + contract_negotiation_id VARCHAR, + created_at BIGINT NOT NULL +); + diff --git a/deployment/terraform/db-scripts/init.sql b/deployment/terraform/db-scripts/init.sql deleted file mode 100644 index 7508e983..00000000 --- a/deployment/terraform/db-scripts/init.sql +++ /dev/null @@ -1,273 +0,0 @@ --- --- Copyright (c) 2022 Daimler TSS GmbH --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- Daimler TSS GmbH - Initial SQL Query --- - --- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES - --- table: edc_asset -CREATE TABLE IF NOT EXISTS edc_asset -( - asset_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - properties JSON DEFAULT '{}', - private_properties JSON DEFAULT '{}', - data_address JSON DEFAULT '{}', - PRIMARY KEY (asset_id) -); - -COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; - - - --- --- Copyright (c) 2022 Daimler TSS GmbH --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- Daimler TSS GmbH - Initial SQL Query --- Microsoft Corporation - refactoring --- - --- table: edc_contract_definitions --- only intended for and tested with H2 and Postgres! -CREATE TABLE IF NOT EXISTS edc_contract_definitions -( - created_at BIGINT NOT NULL, - contract_definition_id VARCHAR NOT NULL, - access_policy_id VARCHAR NOT NULL, - contract_policy_id VARCHAR NOT NULL, - assets_selector JSON NOT NULL, - private_properties JSON, - PRIMARY KEY (contract_definition_id) -); - - --- Statements are designed for and tested with Postgres only! - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER DEFAULT 60000 NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); - - - -CREATE TABLE IF NOT EXISTS edc_contract_agreement -( - agr_id VARCHAR NOT NULL - CONSTRAINT contract_agreement_pk - PRIMARY KEY, - provider_agent_id VARCHAR, - consumer_agent_id VARCHAR, - signing_date BIGINT, - start_date BIGINT, - end_date INTEGER, - asset_id VARCHAR NOT NULL, - policy JSON -); - - -CREATE TABLE IF NOT EXISTS edc_contract_negotiation -( - id VARCHAR NOT NULL - CONSTRAINT contract_negotiation_pk - PRIMARY KEY, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - correlation_id VARCHAR, - counterparty_id VARCHAR NOT NULL, - counterparty_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - type VARCHAR NOT NULL, - state INTEGER DEFAULT 0 NOT NULL, - state_count INTEGER DEFAULT 0, - state_timestamp BIGINT, - error_detail VARCHAR, - agreement_id VARCHAR - CONSTRAINT contract_negotiation_contract_agreement_id_fk - REFERENCES edc_contract_agreement, - contract_offers JSON, - callback_addresses JSON, - trace_context JSON, - pending BOOLEAN DEFAULT FALSE, - protocol_messages JSON, - lease_id VARCHAR - CONSTRAINT contract_negotiation_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; - - -CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index - ON edc_contract_negotiation (correlation_id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex - ON edc_contract_negotiation (id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex - ON edc_contract_agreement (agr_id); - - - - --- --- Copyright (c) 2022 ZF Friedrichshafen AG --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- ZF Friedrichshafen AG - Initial SQL Query --- - --- Statements are designed for and tested with Postgres only! - --- table: edc_policydefinitions -CREATE TABLE IF NOT EXISTS edc_policydefinitions -( - policy_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - permissions JSON, - prohibitions JSON, - duties JSON, - extensible_properties JSON, - inherits_from VARCHAR, - assigner VARCHAR, - assignee VARCHAR, - target VARCHAR, - policy_type VARCHAR NOT NULL, - private_properties JSON, - PRIMARY KEY (policy_id) -); - -COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex - ON edc_policydefinitions (policy_id); - - --- Statements are designed for and tested with Postgres only! - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - -CREATE TABLE IF NOT EXISTS edc_transfer_process -( - transferprocess_id VARCHAR NOT NULL - CONSTRAINT transfer_process_pk - PRIMARY KEY, - type VARCHAR NOT NULL, - state INTEGER NOT NULL, - state_count INTEGER DEFAULT 0 NOT NULL, - state_time_stamp BIGINT, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - trace_context JSON, - error_detail VARCHAR, - resource_manifest JSON, - provisioned_resource_set JSON, - content_data_address JSON, - deprovisioned_resources JSON, - private_properties JSON, - callback_addresses JSON, - pending BOOLEAN DEFAULT FALSE, - transfer_type VARCHAR, - protocol_messages JSON, - lease_id VARCHAR - CONSTRAINT transfer_process_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; - - -CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex - ON edc_transfer_process (transferprocess_id); - -CREATE TABLE IF NOT EXISTS edc_data_request -( - datarequest_id VARCHAR NOT NULL - CONSTRAINT data_request_pk - PRIMARY KEY, - process_id VARCHAR NOT NULL, - connector_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - connector_id VARCHAR, - asset_id VARCHAR NOT NULL, - contract_id VARCHAR NOT NULL, - data_destination JSON NOT NULL, - transfer_process_id VARCHAR NOT NULL - CONSTRAINT data_request_transfer_process_id_fk - REFERENCES edc_transfer_process - ON UPDATE RESTRICT ON DELETE CASCADE -); - - -COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex - ON edc_data_request (datarequest_id); - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); \ No newline at end of file diff --git a/deployment/terraform/db-scripts/main.tf b/deployment/terraform/db-scripts/main.tf new file mode 100644 index 00000000..830a26cc --- /dev/null +++ b/deployment/terraform/db-scripts/main.tf @@ -0,0 +1,6 @@ +resource "null_resource" "db-scripts" { + provisioner "local-exec" { + command = "${path.module}/db-scripts.sh" + interpreter = ["bash", "-c"] + } +} \ No newline at end of file diff --git a/deployment/terraform/db-scripts/policy-definition-store/schema.sql b/deployment/terraform/db-scripts/policy-definition-store/schema.sql new file mode 100644 index 00000000..d4ef8127 --- /dev/null +++ b/deployment/terraform/db-scripts/policy-definition-store/schema.sql @@ -0,0 +1,41 @@ +-- +-- Copyright (c) 2022 ZF Friedrichshafen AG +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- ZF Friedrichshafen AG - Initial SQL Query +-- + +-- Statements are designed for and tested with Postgres only! + +-- table: edc_policydefinitions +CREATE TABLE IF NOT EXISTS edc_policydefinitions +( + policy_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + permissions JSON, + prohibitions JSON, + duties JSON, + extensible_properties JSON, + inherits_from VARCHAR, + assigner VARCHAR, + assignee VARCHAR, + target VARCHAR, + policy_type VARCHAR NOT NULL, + private_properties JSON, + PRIMARY KEY (policy_id) +); + +COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; + +CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex + ON edc_policydefinitions (policy_id); diff --git a/deployment/terraform/db-scripts/transfer-process-store/schema.sql b/deployment/terraform/db-scripts/transfer-process-store/schema.sql new file mode 100644 index 00000000..ab16436c --- /dev/null +++ b/deployment/terraform/db-scripts/transfer-process-store/schema.sql @@ -0,0 +1,70 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_transfer_process +( + transferprocess_id VARCHAR NOT NULL + CONSTRAINT transfer_process_pk + PRIMARY KEY, + type VARCHAR NOT NULL, + state INTEGER NOT NULL, + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + trace_context JSON, + error_detail VARCHAR, + resource_manifest JSON, + provisioned_resource_set JSON, + content_data_address JSON, + deprovisioned_resources JSON, + private_properties JSON, + callback_addresses JSON, + pending BOOLEAN DEFAULT FALSE, + transfer_type VARCHAR, + protocol_messages JSON, + data_plane_id VARCHAR, + correlation_id VARCHAR, + counter_party_address VARCHAR, + protocol VARCHAR, + asset_id VARCHAR, + contract_id VARCHAR, + data_destination JSON, + lease_id VARCHAR + CONSTRAINT transfer_process_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; + + +CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex + ON edc_transfer_process (transferprocess_id); + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS transfer_process_state ON edc_transfer_process (state,state_time_stamp); \ No newline at end of file diff --git a/deployment/terraform/deploy-services.sh b/deployment/terraform/deploy-services.sh index 1b6e3a0d..d533142e 100755 --- a/deployment/terraform/deploy-services.sh +++ b/deployment/terraform/deploy-services.sh @@ -145,28 +145,22 @@ if [ "$TF_VAR_persistence_type" == "PostgreSQLaaS" ]; then fi if [ "$TF_VAR_persistence_type" == "PostgreSQL" ]; then - echo "Deploying postgres" - helm repo add bitnami https://charts.bitnami.com/bitnami - set +e - helm --kubeconfig=$TF_VAR_kubeconfig install postgres bitnami/postgresql -n $TF_VAR_namespace --set global.postgresql.auth.username=$TF_VAR_pg_username --set global.postgresql.auth.password=$TF_VAR_pg_password --set global.postgresql.auth.database=$TF_VAR_pg_database - set -e - - kubectl --kubeconfig=$TF_VAR_kubeconfig wait --for=condition=Ready=True pod -l app.kubernetes.io/name=postgresql -n $TF_VAR_namespace --timeout=600s + echo "Deploying postgresql" + # Create PostgreSQL instance + cd ../postgresql-deploy + terraform init + terraform apply -auto-approve - export TF_VAR_pg_host="postgres-postgresql" + export TF_VAR_pg_host="postgresql."$TF_VAR_namespace fi # Create the database if [ "$TF_VAR_persistence_type" == "PostgreSQLaaS" ] || [ "$TF_VAR_persistence_type" == "PostgreSQL" ]; then + echo "Running database scripts" + # Run scripts to create database schemas cd ../db-scripts - echo "Creating database $TF_VAR_pg_database" - set +e - kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="postgres" --command="CREATE DATABASE $TF_VAR_pg_database;" - set -e - - kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-restore-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ../db-scripts/init.sql -else - echo "WARNING: No persistence, the data will be lost if container pods are restarted" + terraform init + terraform apply -auto-approve fi echo "Deploying ionos s3" diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf index 8f74f22c..be8bd1d8 100644 --- a/deployment/terraform/ionos-s3-deploy/main.tf +++ b/deployment/terraform/ionos-s3-deploy/main.tf @@ -16,11 +16,6 @@ variable "ids_webhook_address" { default = "http://localhost:8282" } -variable "persistence_type" { - type = string - default = "None" -} - variable "image_repository" { type = string default = "ghcr.io/digital-ecosystems/connector" @@ -56,6 +51,7 @@ variable "pg_password" { default = "postgres" } +variable "s3_endpoint_region" {} variable "ionos_token" {} variable "vaultname" { @@ -63,7 +59,7 @@ variable "vaultname" { } locals { - root_token = fileexists("../vault-init/vault-keys.json") ? "${jsondecode(file("../vault-init/vault-keys.json")).root_token}" : "" + vault_token = fileexists("../vault-init/vault-tokens.json") ? "${jsondecode(file("../vault-init/vault-tokens.json")).auth.client_token}" : "" } resource "helm_release" "edc-ionos-s3" { @@ -77,7 +73,7 @@ resource "helm_release" "edc-ionos-s3" { set { name = "edc.vault.hashicorp.token" - value = "${jsondecode(file("../vault-init/vault-keys.json")).root_token}" + value = local.vault_token } values = [ @@ -89,11 +85,6 @@ resource "helm_release" "edc-ionos-s3" { value = "http://${var.vaultname}:8200" } - set { - name = "edc.vault.hashicorp.token" - value = local.root_token - } - set { name = "edc.ionos.endpoint.region" value = var.s3_endpoint_region diff --git a/deployment/terraform/postgresql-deploy/main.tf b/deployment/terraform/postgresql-deploy/main.tf new file mode 100644 index 00000000..57c9a9c7 --- /dev/null +++ b/deployment/terraform/postgresql-deploy/main.tf @@ -0,0 +1,51 @@ +provider "helm" { + kubernetes { + config_path = "${var.kubeconfig}" + } +} + +variable "kubeconfig" { + type = string +} + +variable "namespace" { + default = "edc-ionos-s3" +} + +variable "pg_username" { + type = string + default = "postgres" +} + +variable "pg_password" { + type = string + default = "postgres" +} + +variable "pg_database" { + type = string + default = "postgres" +} + +resource "helm_release" "postgresql" { + name = "postgresql" + repository = "https://charts.bitnami.com/bitnami" + chart = "postgresql" + + namespace = var.namespace + + set { + name = "global.postgresql.auth.username" + value = var.pg_username + } + + set { + name = "global.postgresql.auth.password" + value = var.pg_password + } + + set { + name = "global.postgresql.auth.database" + value = var.pg_database + } +} \ No newline at end of file diff --git a/deployment/terraform/destroy-services.sh b/deployment/terraform/undeploy-services.sh similarity index 71% rename from deployment/terraform/destroy-services.sh rename to deployment/terraform/undeploy-services.sh index b7ba48a2..ba093c80 100755 --- a/deployment/terraform/destroy-services.sh +++ b/deployment/terraform/undeploy-services.sh @@ -21,11 +21,21 @@ cd ../ionos-s3-deploy terraform init terraform destroy -auto-approve -# Destroy Ionos Postgres Cluister +# Destroy ionos postgresql cluster cd ../ionos-postgresqlaas terraform init terraform destroy -auto-approve +# Destroy postgresql +cd ../postgresql-deploy +terraform init +terraform destroy -auto-approve + +# Destroy db-scripts +cd ../db-scripts +terraform init +terraform destroy -auto-approve + cd ../ # remove terraform state @@ -54,6 +64,18 @@ rm -f ./ionos-postgresqlaas/terraform.tfstate rm -f ./ionos-postgresqlaas/.terraform.lock.hcl rm -f ./ionos-postgresqlaas/terraform.tfstate.backup +rm -rf ./postgresql-deploy/.terraform +rm -f ./postgresql-deploy/terraform.tfstate +rm -f ./postgresql-deploy/.terraform.lock.hcl +rm -f ./postgresql-deploy/terraform.tfstate.backup + +rm -rf ./db-scripts/.terraform +rm -f ./db-scripts/terraform.tfstate +rm -f ./db-scripts/.terraform.lock.hcl +rm -f ./db-scripts/terraform.tfstate.backup + rm -f vault-init/vault-keys.json -helm uninstall postgres -n $TF_VAR_namespace +rm -f vault-init/vault-tokens.json kubectl --kubeconfig $TF_VAR_kubeconfig delete namespace $TF_VAR_namespace + +echo "Undeployment complete" \ No newline at end of file diff --git a/deployment/terraform/vault-deploy/main.tf b/deployment/terraform/vault-deploy/main.tf index 6ad0bc64..28c7d1d8 100644 --- a/deployment/terraform/vault-deploy/main.tf +++ b/deployment/terraform/vault-deploy/main.tf @@ -32,7 +32,7 @@ resource "helm_release" "vault" { repository = "https://helm.releases.hashicorp.com" chart = "vault" - version = "v0.19.0" + version = "v0.28.1" namespace = var.namespace create_namespace = true diff --git a/deployment/terraform/vault-init/certs/private.pem b/deployment/terraform/vault-init/certs/private.pem new file mode 100644 index 00000000..81c28bac --- /dev/null +++ b/deployment/terraform/vault-init/certs/private.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIARDUGJgKy1yzxkueIJ1k3MPUWQ/tbQWQNqW6TjyHpdcoAoGCCqGSM49 +AwEHoUQDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5roYnkAXuqCYfNK3ex+hMWFuiX +GUxHlzShAehR6wvwzV23bbC0tcFcVgW//A== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/deployment/terraform/vault-init/certs/public.pem b/deployment/terraform/vault-init/certs/public.pem new file mode 100644 index 00000000..977a1957 --- /dev/null +++ b/deployment/terraform/vault-init/certs/public.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5r +oYnkAXuqCYfNK3ex+hMWFuiXGUxHlzShAehR6wvwzV23bbC0tcFcVgW//A== +-----END PUBLIC KEY----- \ No newline at end of file diff --git a/deployment/terraform/vault-init/vault-init.sh b/deployment/terraform/vault-init/vault-init.sh index 18da804b..6931f968 100755 --- a/deployment/terraform/vault-init/vault-init.sh +++ b/deployment/terraform/vault-init/vault-init.sh @@ -30,14 +30,19 @@ kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR # Login to Vault kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault login $(jq -r ".root_token" vault-keys.json) - if [[ "$INITIALIZED" == "false" ]]; then # Enable KV secrets engine kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault secrets enable -version=2 -path=secret kv fi +## Create connector token +kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault token create -policy=root -renewable=true -ttl=300s -format=json > vault-tokens.json + # Add secrets to Vault kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.access.key content=$TF_VAR_s3_access_key kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.secret.key content=$TF_VAR_s3_secret_key kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint.region content=$TF_VAR_s3_endpoint_region kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.token content=$TF_VAR_ionos_token + +kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.private.key content="$(cat ./certs/private.pem)" +kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.public.key content="$(cat ./certs/public.pem)" diff --git a/deployment/terraform/vault-keys.json b/deployment/terraform/vault-keys.json deleted file mode 100644 index 7727a188..00000000 --- a/deployment/terraform/vault-keys.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "root_token": "" -} diff --git a/extensions/build.gradle.kts b/extensions/build.gradle.kts index f0bdea98..93af45b3 100644 --- a/extensions/build.gradle.kts +++ b/extensions/build.gradle.kts @@ -3,12 +3,6 @@ plugins { `maven-publish` } -repositories { - mavenLocal() - mavenCentral() - -} - configure { publications { withType(MavenPublication::class.java) { @@ -21,6 +15,11 @@ configure { } } developers { + developer { + id.set("jannotti-glaucio") + name.set("Glaucio Jannotti") + email.set("glaucio.jannotti@ionos.com") + } developer { id.set("paulolory-ionos") name.set("Paulo Lory") diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java similarity index 70% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java index 8947fcd8..38761d76 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/S3CoreExtension.java @@ -12,10 +12,10 @@ * */ -package com.ionos.edc.extension.s3.configuration; +package com.ionos.edc.extension.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3ConnectorApiImpl; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.connector.S3ConnectorImpl; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provides; @@ -25,13 +25,13 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_ACCESS_KEY; -import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_SECRET_KEY; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_REGION; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_SECRET_KEY; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_TOKEN; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; -@Provides(S3ConnectorApi.class) +@Provides(S3Connector.class) @Extension(value = S3CoreExtension.NAME) public class S3CoreExtension implements ServiceExtension { @@ -56,17 +56,17 @@ public void initialize(ServiceExtensionContext context) { var region = vault.resolveSecret(IONOS_REGION); var token = vault.resolveSecret(IONOS_TOKEN); - if(accessKey == null || secretKey == null || region == null || token == null) { + if(accessKey == null || secretKey == null || region ==null || token == null) { monitor.warning("Couldn't connect or the vault didn't return values, falling back to ConfigMap Configuration"); - accessKey = context.getSetting(IONOS_ACCESS_KEY, IONOS_ACCESS_KEY); - secretKey = context.getSetting(IONOS_SECRET_KEY, IONOS_SECRET_KEY); - region = context.getSetting(IONOS_REGION, IONOS_REGION); - token = context.getSetting(IONOS_TOKEN, IONOS_TOKEN); + accessKey = context.getSetting(IONOS_ACCESS_KEY, null); + secretKey = context.getSetting(IONOS_SECRET_KEY, null); + region = context.getSetting(IONOS_REGION, null); + token = context.getSetting(IONOS_TOKEN, null); } - var maxFiles = context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT); + var maxFiles = Integer.valueOf(context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT)); - var s3Api = new S3ConnectorApiImpl(region, accessKey, secretKey, token, maxFiles); - context.registerService(S3ConnectorApi.class, s3Api); + var s3Connector = new S3ConnectorImpl(region, accessKey, secretKey, token, maxFiles); + context.registerService(S3Connector.class, s3Connector); } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java similarity index 65% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java index 738e2872..28955004 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3AccessKey.java @@ -1,4 +1,18 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; +/* + * Copyright (c) 2024 IONOS + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * IONOS + * + */ + +package com.ionos.edc.extension.s3.api; public class S3AccessKey { public static final String AVAILABLE_STATUS = "AVAILABLE"; diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java similarity index 60% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java index dad960b8..893a894f 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ApiClient.java @@ -1,4 +1,18 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; +/* + * Copyright (c) 2024 IONOS + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * IONOS + * + */ + +package com.ionos.edc.extension.s3.api; import java.io.IOException; @@ -6,52 +20,63 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.*; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; import org.eclipse.edc.spi.EdcException; -public class S3ApiConnector { +public class S3ApiClient { private static final String BASE_URL = "https://s3.ionos.com"; + private static final String REGIONS_ENDPOINT_URL = BASE_URL + "/regions"; + private static final String ACCESS_KEYS_ENDPOINT_URL = BASE_URL + "/accesskeys"; + + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String BEARER_TOKEN_PREFIX = "Bearer "; + private static final String JSON_MEDIA_TYPE = "application/json"; private final OkHttpClient client; private final ObjectMapper objectMapper; - public S3ApiConnector() { + public S3ApiClient() { client = new OkHttpClient(); objectMapper = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } public S3Regions retrieveRegions(String token) { - String url = BASE_URL + "/regions"; - Request request = new Request.Builder().url(url) - .addHeader("Authorization", "Bearer " + token) + Request request = new Request.Builder().url(REGIONS_ENDPOINT_URL) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) .get() .build(); + try (Response response = client.newCall(request).execute()) { if (!response.isSuccessful()) { throw new EdcException("Unexpected code [" + response.code() + "] retrieving S3 regions"); } + if (response.body() == null) throw new IOException("Empty response body retrieving S3 regions"); else return objectMapper.readValue(response.body().string(), new TypeReference() {}); + } catch (IOException e) { throw new EdcException("Error retrieving S3 accesskey", e); } } public S3AccessKey createAccessKey(String token) { - String url = BASE_URL + "/accesskeys"; - Request request = new Request.Builder().url(url) - .addHeader("Authorization", "Bearer " + token) - .post(RequestBody.create(MediaType.get("application/json"), new byte[0])) + Request request = new Request.Builder().url(ACCESS_KEYS_ENDPOINT_URL) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) + .post(RequestBody.create(MediaType.get(JSON_MEDIA_TYPE), new byte[0])) .build(); try (Response response = client.newCall(request).execute()) { if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] creating S3 accesskey"); + throw new EdcException("Unexpected code [" + response.code() + "] creating S3 accesskey"); } if (response.body() == null) @@ -65,16 +90,16 @@ public S3AccessKey createAccessKey(String token) { } public S3AccessKey retrieveAccessKey(String token, String keyID) { - String url = BASE_URL + "/accesskeys/" + keyID; + String url = ACCESS_KEYS_ENDPOINT_URL + "/" + keyID; Request request = new Request.Builder().url(url) - .addHeader("Authorization", "Bearer " + token) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) .get() .build(); try (Response response = client.newCall(request).execute()) { if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] retrieving S3 accesskey"); + throw new EdcException("Unexpected code [" + response.code() + "] retrieving S3 accesskey"); } if (response.body() == null) @@ -88,17 +113,16 @@ public S3AccessKey retrieveAccessKey(String token, String keyID) { } public void deleteAccessKey(String token, String keyID) { - String url = BASE_URL + "/accesskeys/" + keyID; + String url = ACCESS_KEYS_ENDPOINT_URL + "/" + keyID; Request request = new Request.Builder().url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) + .addHeader(AUTHORIZATION_HEADER, BEARER_TOKEN_PREFIX + token) .delete() .build(); try (Response response = client.newCall(request).execute()) { if (!response.isSuccessful()) { - throw new IOException("Unexpected code [" + response + "] deleting S3 accesskey"); + throw new EdcException("Unexpected code [" + response.code() + "] deleting S3 accesskey"); } } catch (IOException e) { throw new EdcException("Error deleting S3 accesskey", e); diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Region.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java similarity index 54% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Region.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java index 507c731e..17d5b109 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Region.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Region.java @@ -11,28 +11,26 @@ * IONOS * */ -package com.ionos.edc.extension.s3.connector.ionosapi; + +package com.ionos.edc.extension.s3.api; public class S3Region { private String id; - private Properties properties; - public String getId() { - return id; - } - - public Properties getProperties() { - return properties; - } - - public static class Properties { + public String getId() { + return id; + } + public Properties getProperties() { + return properties; + } - private String endpoint; + public static class Properties { + private String endpoint; - public String getEndpoint() { - return endpoint; - } - } + public String getEndpoint() { + return endpoint; + } + } } \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Regions.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Regions.java similarity index 60% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Regions.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Regions.java index 5ebfebfa..5b9d0154 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Regions.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Regions.java @@ -11,19 +11,20 @@ * IONOS * */ -package com.ionos.edc.extension.s3.connector.ionosapi; + +package com.ionos.edc.extension.s3.api; import java.util.List; public class S3Regions { - private List items; + private List items; - public List getItems() { - return items; - } + public List getItems() { + return items; + } - public void setItems(List items) { - this.items = items; - } + public void setItems(List items) { + this.items = items; + } } \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java deleted file mode 100644 index 9413feed..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/MinioConnector.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022 IONOS - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * IONOS - * - */ - -package com.ionos.edc.extension.s3.connector; - -import io.minio.MinioClient; - -public class MinioConnector { - - public MinioClient connect(String endpoint, String accessKey, String secretKey) { - return MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); - } -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java similarity index 80% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java index 6307e128..495e1c30 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3Connector.java @@ -12,16 +12,17 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.connector; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3AccessKey; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; import java.io.ByteArrayInputStream; import java.util.List; @ExtensionPoint -public interface S3ConnectorApi { +public interface S3Connector { void createBucket(String bucketName); @@ -41,5 +42,5 @@ public interface S3ConnectorApi { void deleteAccessKey(String keyID); - S3ConnectorApi clone(String region, String accessKey, String secretKey); + S3Connector clone(String region, String accessKey, String secretKey); } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java similarity index 80% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java index b6491b6b..7d82e4ee 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/S3ConnectorImpl.java @@ -12,13 +12,13 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.connector; -import com.ionos.edc.extension.s3.connector.MinioConnector; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; -import com.ionos.edc.extension.s3.connector.ionosapi.S3ApiConnector; +import com.ionos.edc.extension.s3.api.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3ApiClient; -import com.ionos.edc.extension.s3.connector.ionosapi.S3Region; +import com.ionos.edc.extension.s3.api.S3Region; +import com.ionos.edc.extension.s3.types.S3Object; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; @@ -36,24 +36,23 @@ import static com.ionos.edc.extension.s3.schema.IonosBucketSchema.REGION_ID_DEFAULT; -public class S3ConnectorApiImpl implements S3ConnectorApi { +public class S3ConnectorImpl implements S3Connector { - MinioConnector miniConnector = new MinioConnector(); - S3ApiConnector ionoss3Api = new S3ApiConnector(); + private final S3ApiClient S3ApiClient = new S3ApiClient(); private final MinioClient minioClient; private final String regionId; private final String token; - private final Integer maxFiles; + private final int maxFiles; - public S3ConnectorApiImpl(String regionId, @NotNull String accessKey, @NotNull String secretKey, @NotNull String token, int maxFiles) { + public S3ConnectorImpl(String regionId, @NotNull String accessKey, @NotNull String secretKey, @NotNull String token, int maxFiles) { this.token = token; this.maxFiles = maxFiles; this.regionId = Objects.requireNonNullElse(regionId, REGION_ID_DEFAULT); var endpoint = getEndpoint( this.regionId , token); - this.minioClient = miniConnector.connect(endpoint, accessKey, secretKey); + this.minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); } @Override @@ -166,32 +165,32 @@ public List listObjects(String bucketName, String objectName) { @Override public S3AccessKey createAccessKey() { try{ - return ionoss3Api.createAccessKey(token); + return S3ApiClient.createAccessKey(token); } catch (Exception e) { - throw new EdcException("Creating temporary key - (Warning: max 5 keys on the storage) - " + e.getMessage()); + throw new EdcException("Error creating access key", e); } } @Override public S3AccessKey retrieveAccessKey(String keyID) { try{ - return ionoss3Api.retrieveAccessKey(token, keyID); + return S3ApiClient.retrieveAccessKey(token, keyID); } catch (Exception e) { - throw new EdcException("Retrieving temporary key: " + e.getMessage()); + throw new EdcException("Error retrieving access key", e); } } @Override public void deleteAccessKey(String keyID) { try{ - ionoss3Api.deleteAccessKey(token, keyID); + S3ApiClient.deleteAccessKey(token, keyID); } catch (Exception e) { - throw new EdcException("Deleting temporary key: " + e.getMessage()); + throw new EdcException("Error deleting access key", e); } } private String getEndpoint(String regionId, String token) { - var regions = ionoss3Api.retrieveRegions(token); + var regions = S3ApiClient.retrieveRegions(token); for (S3Region region: regions.getItems()) { if (region.getId().equals(regionId)) { @@ -202,8 +201,7 @@ private String getEndpoint(String regionId, String token) { } @Override - public S3ConnectorApi clone(String region, String accessKey, String secretKey) { - return new S3ConnectorApiImpl(region, accessKey, secretKey, this.token, this.maxFiles); + public S3Connector clone(String region, String accessKey, String secretKey) { + return new S3ConnectorImpl(region, accessKey, secretKey, this.token, this.maxFiles); } - } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java index 65921a56..740b3dd9 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java @@ -14,7 +14,9 @@ package com.ionos.edc.extension.s3.schema; -import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; +import org.eclipse.edc.spi.types.domain.transfer.FlowType; + +import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; public interface IonosBucketSchema { String TYPE = "IonosS3"; @@ -27,5 +29,7 @@ public interface IonosBucketSchema { String ACCESS_KEY_ID = EDC_NAMESPACE + "accessKey"; String SECRET_ACCESS_KEY = EDC_NAMESPACE + "secretKey"; + String PUSH_TRANSFER_TYPE = TYPE + "-" + FlowType.PUSH; + String REGION_ID_DEFAULT = "de"; } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java similarity index 90% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java index 5396e595..1ee51bf3 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/IonosToken.java @@ -12,12 +12,12 @@ * */ -package com.ionos.edc.extension.s3.configuration; +package com.ionos.edc.extension.s3.types; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import org.eclipse.edc.connector.transfer.spi.types.SecretToken; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.SecretToken; @JsonTypeName("dataspaceconnector:ionostoken") public class IonosToken implements SecretToken { diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java similarity index 95% rename from extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java rename to extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java index 16bf41fd..7cd19e70 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3Object.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/types/S3Object.java @@ -12,7 +12,7 @@ * */ -package com.ionos.edc.extension.s3.api; +package com.ionos.edc.extension.s3.types; public record S3Object(String objectName, long size) { diff --git a/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 7ac275e9..a9890db8 100644 --- a/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/extensions/core-ionos-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -12,5 +12,5 @@ # # -com.ionos.edc.extension.s3.configuration.S3CoreExtension +com.ionos.edc.extension.s3.S3CoreExtension diff --git a/extensions/data-plane-ionos-s3/build.gradle.kts b/extensions/data-plane-ionos-s3/build.gradle.kts index 3d166d94..0e11ae1a 100644 --- a/extensions/data-plane-ionos-s3/build.gradle.kts +++ b/extensions/data-plane-ionos-s3/build.gradle.kts @@ -3,9 +3,9 @@ plugins { `maven-publish` } -val javaVersion: String by project val edcGroup: String by project val edcVersion: String by project +val metaModelVersion: String by project val extensionsGroup: String by project val extensionsVersion: String by project val junitVersion: String by project @@ -17,16 +17,16 @@ val gitHubUser: String? by project val gitHubToken: String? by project dependencies { - api("${edcGroup}:data-plane-spi:${edcVersion}") + api("${edcGroup}:runtime-metamodel:${metaModelVersion}") - implementation(project(":extensions:core-ionos-s3")) - implementation("${edcGroup}:util:${edcVersion}") + implementation("${edcGroup}:util-lib:${edcVersion}") implementation("${edcGroup}:transfer-spi:${edcVersion}") - implementation("${edcGroup}:data-plane-util:${edcVersion}") - implementation("${edcGroup}:data-plane-core:${edcVersion}") - implementation("${edcGroup}:http:${edcVersion}") implementation("${edcGroup}:validator-spi:${edcVersion}") - + implementation("${edcGroup}:data-plane-util:${edcVersion}") + + implementation(project(":extensions:core-ionos-s3")) + + testImplementation("${edcGroup}:junit:${edcVersion}") testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") testImplementation("org.mockito:mockito-core:${mockitoVersion}") diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java index 29ec30b1..17b6c9b0 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java @@ -14,7 +14,7 @@ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer; import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; import org.eclipse.edc.runtime.metamodel.annotation.Extension; @@ -32,7 +32,7 @@ public class DataPlaneIonosS3Extension implements ServiceExtension { private PipelineService pipelineService; @Inject - private S3ConnectorApi s3Api; + private S3Connector s3Connector; @Inject private DataTransferExecutorServiceContainer executorContainer; @@ -52,10 +52,10 @@ public String name() { public void initialize(ServiceExtensionContext context) { var monitor = context.getMonitor(); - var sourceFactory = new IonosDataSourceFactory(s3Api, monitor); + var sourceFactory = new IonosDataSourceFactory(s3Connector, monitor); pipelineService.registerFactory(sourceFactory); - var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager); + var sinkFactory = new IonosDataSinkFactory(s3Connector, executorContainer.getExecutorService(), monitor, vault, typeManager); pipelineService.registerFactory(sinkFactory); context.getMonitor().info("File Transfer Extension initialized!"); } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java index 259645ca..edc1a547 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java @@ -14,7 +14,7 @@ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; @@ -30,7 +30,7 @@ public class IonosDataSink extends ParallelSink { - private S3ConnectorApi s3Api; + private S3Connector s3Connector; private String bucketName; private String path; @@ -81,7 +81,7 @@ protected StreamResult transferParts(List parts) { var byteArray = streamsOutput.toByteArray(); try (var streamsInput = new ByteArrayInputStream(byteArray)) { - s3Api.uploadObject(bucketName, blobName, streamsInput); + s3Connector.uploadObject(bucketName, blobName, streamsInput); streamsOutput.close(); } catch (Exception e) { @@ -122,8 +122,8 @@ public static Builder newInstance() { return new Builder(); } - public Builder s3Api(S3ConnectorApi s3Api) { - sink.s3Api = s3Api; + public Builder s3Connector(S3Connector s3Connector) { + sink.s3Connector = s3Connector; return this; } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java index 8bb1f057..b9d6dd38 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java @@ -15,8 +15,8 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.validation.IonosSinkDataAddressValidationRule; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; @@ -26,7 +26,7 @@ import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.types.TypeManager; import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; +import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; import org.eclipse.edc.validator.spi.Validator; import org.eclipse.edc.validator.spi.ValidationResult; import org.jetbrains.annotations.NotNull; @@ -37,14 +37,14 @@ public class IonosDataSinkFactory implements DataSinkFactory { private final ExecutorService executorService; private final Monitor monitor; - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Vault vault; private final TypeManager typeManager; private final Validator validator = new IonosSinkDataAddressValidationRule(); - public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager) { - this.s3Api = s3Api; + public IonosDataSinkFactory(S3Connector s3Connector, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager) { + this.s3Connector = s3Connector; this.executorService = executorService; this.monitor = monitor; this.vault = vault; @@ -52,18 +52,18 @@ public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorServic } @Override - public boolean canHandle(DataFlowRequest request) { - return IonosBucketSchema.TYPE.equals(request.getDestinationDataAddress().getType()); + public String supportedType() { + return IonosBucketSchema.TYPE; } @Override - public @NotNull Result validateRequest(DataFlowRequest request) { + public @NotNull Result validateRequest(DataFlowStartMessage request) { var destination = request.getDestinationDataAddress(); return validator.validate(destination).flatMap(ValidationResult::toResult); } @Override - public DataSink createSink(DataFlowRequest request) { + public DataSink createSink(DataFlowStartMessage request) { var validationResult = validateRequest(request); if (validationResult.failed()) { @@ -80,7 +80,7 @@ public DataSink createSink(DataFlowRequest request) { var region = destination.getStringProperty(IonosBucketSchema.REGION_ID); - var s3ApiTemp = s3Api.clone(region, token.getAccessKey(), token.getSecretKey()); + var s3ConnectorTemp = s3Connector.clone(region, token.getAccessKey(), token.getSecretKey()); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) @@ -88,7 +88,7 @@ public DataSink createSink(DataFlowRequest request) { .requestId(request.getId()) .executorService(executorService) .monitor(monitor) - .s3Api(s3ApiTemp) + .s3Connector(s3ConnectorTemp) .build(); } } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java index 7a7bd2b8..9f683b94 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java @@ -15,8 +15,8 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3Object; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; @@ -36,7 +36,7 @@ class IonosDataSource implements DataSource { - private S3ConnectorApi s3Api; + private S3Connector s3Connector; private Monitor monitor; private String bucketName; private String blobName; @@ -49,7 +49,7 @@ private IonosDataSource() { @Override public StreamResult> openPartStream() { - var objects = s3Api.listObjects(this.bucketName, this.blobName); + var objects = s3Connector.listObjects(this.bucketName, this.blobName); if (objects.isEmpty()) { return failure(new StreamFailure( @@ -76,7 +76,7 @@ public StreamResult> openPartStream() { } List parts = objects.stream() - .map(object -> new S3Part(s3Api, monitor, bucketName, object.objectName(), object.isDirectory(), object.size())) + .map(object -> new S3Part(s3Connector, monitor, bucketName, object.objectName(), object.isDirectory(), object.size())) .collect(Collectors.toList()); return success(parts.stream()); } @@ -99,7 +99,8 @@ public void close() { } public static class S3Part implements Part { - private final S3ConnectorApi s3Api; + + private final S3Connector s3Connector; private final Monitor monitor; private final String bucketName; private final String blobName; @@ -109,9 +110,9 @@ public static class S3Part implements Part { private boolean isOpened = true; private long currentOffset = 0; - S3Part(S3ConnectorApi s3Api, Monitor monitor, String bucketName, String blobName, boolean isDirectory, long fileSize) { + S3Part(S3Connector s3Connector, Monitor monitor, String bucketName, String blobName, boolean isDirectory, long fileSize) { super(); - this.s3Api = s3Api; + this.s3Connector = s3Connector; this.monitor = monitor; this.bucketName = bucketName; this.blobName = blobName; @@ -141,9 +142,9 @@ public InputStream openStream() { InputStream stream; if (isDirectory || (fileSize <= chunkSize())) { - stream = s3Api.getObject(bucketName, blobName); + stream = s3Connector.getObject(bucketName, blobName); } else { - stream = s3Api.getObject(bucketName, blobName, currentOffset, chunkSize()); + stream = s3Connector.getObject(bucketName, blobName, currentOffset, chunkSize()); } if (!isDirectory) { @@ -174,8 +175,8 @@ public static Builder newInstance() { return new Builder(); } - public Builder client(S3ConnectorApi s3Api) { - source.s3Api = s3Api; + public Builder client(S3Connector s3Connector) { + source.s3Connector = s3Connector; return this; } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java index 811afc63..13c95c68 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceFactory.java @@ -15,7 +15,7 @@ package com.ionos.edc.dataplane.ionos.s3; import com.ionos.edc.dataplane.ionos.s3.validation.IonosSourceDataAddressValidationRule; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.connector.S3Connector; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; @@ -23,35 +23,35 @@ import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.result.Result; import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; +import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; import org.eclipse.edc.validator.spi.Validator; import org.eclipse.edc.validator.spi.ValidationResult; import org.jetbrains.annotations.NotNull; public class IonosDataSourceFactory implements DataSourceFactory { - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Monitor monitor; private final Validator validator = new IonosSourceDataAddressValidationRule(); - public IonosDataSourceFactory(S3ConnectorApi s3Api, Monitor monitor) { - this.s3Api = s3Api; + public IonosDataSourceFactory(S3Connector s3Connector, Monitor monitor) { + this.s3Connector = s3Connector; this.monitor = monitor; } @Override - public boolean canHandle(DataFlowRequest request) { - return IonosBucketSchema.TYPE.equals(request.getSourceDataAddress().getType()); + public String supportedType() { + return IonosBucketSchema.TYPE; } @Override - public @NotNull Result validateRequest(DataFlowRequest request) { + public @NotNull Result validateRequest(DataFlowStartMessage request) { var source = request.getSourceDataAddress(); return validator.validate(source).flatMap(ValidationResult::toResult); } @Override - public DataSource createSource(DataFlowRequest request) { + public DataSource createSource(DataFlowStartMessage request) { var validationResult = validateRequest(request); if (validationResult.failed()) { @@ -61,7 +61,7 @@ public DataSource createSource(DataFlowRequest request) { var source = request.getSourceDataAddress(); return IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(source.getStringProperty(IonosBucketSchema.BUCKET_NAME)) .blobName(source.getStringProperty(IonosBucketSchema.BLOB_NAME)) diff --git a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java index df97cfb8..149e5676 100644 --- a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java +++ b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java @@ -3,8 +3,8 @@ */ package com.ionos.edc.dataplane.ionos.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.api.S3Object; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.S3Object; import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; import org.eclipse.edc.spi.monitor.Monitor; import org.junit.jupiter.api.BeforeEach; @@ -37,7 +37,7 @@ class IonosDataSourceTest { private static final String TEST_SUB_FOLDER_2_NAME = "device2/"; @Mock - private S3ConnectorApi s3Api; + private S3Connector s3Connector; @Mock private Monitor monitor; @@ -50,10 +50,10 @@ public void setup() { public void openPartStream_empty() { doReturn(List.of()) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FILE_1_NAME) @@ -70,10 +70,10 @@ public void openPartStream_singleFile() { var s3Objects = List.of(new S3Object(TEST_FILE_1_NAME, TEST_FILE_1_SIZE)); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FILE_1_NAME) @@ -99,10 +99,10 @@ public void openPartStream_folder() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -140,10 +140,10 @@ public void openPartStream_folder_includeFiles() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -178,10 +178,10 @@ public void openPartStream_folder_excludeFiles() { new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -213,10 +213,10 @@ public void openPartStream_subFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -259,10 +259,10 @@ public void openPartStream_subFolder_includeFilesInFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -298,10 +298,10 @@ public void openPartStream_subFolder_excludeFilesInFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -342,10 +342,10 @@ public void openPartStream_subFolder_includeFilesInSubFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) @@ -381,10 +381,10 @@ public void openPartStream_subFolder_excludeFilesInSubFolder() { new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) ); doReturn(s3Objects) - .when(s3Api).listObjects(any(String.class), any(String.class)); + .when(s3Connector).listObjects(any(String.class), any(String.class)); var dataSource = IonosDataSource.Builder.newInstance() - .client(s3Api) + .client(s3Connector) .monitor(monitor) .bucketName(TEST_BUCKET) .blobName(TEST_FOLDER_NAME) diff --git a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java index 0d46d80c..798fd19e 100644 --- a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java +++ b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java @@ -1,6 +1,5 @@ package com.ionos.edc.dataplane.ionos.s3.util; -import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; import org.eclipse.edc.spi.EdcException; import org.junit.jupiter.api.Test; diff --git a/extensions/provision-ionos-s3/build.gradle.kts b/extensions/provision-ionos-s3/build.gradle.kts index 38478452..536cbca3 100644 --- a/extensions/provision-ionos-s3/build.gradle.kts +++ b/extensions/provision-ionos-s3/build.gradle.kts @@ -3,7 +3,6 @@ plugins { `maven-publish` } -val javaVersion: String by project val edcGroup: String by project val edcVersion: String by project val metaModelVersion: String by project @@ -19,8 +18,13 @@ val gitHubToken: String? by project dependencies { api("${edcGroup}:runtime-metamodel:${metaModelVersion}") - implementation(project(":extensions:core-ionos-s3")) implementation("${edcGroup}:transfer-spi:${edcVersion}") + + implementation(project(":extensions:core-ionos-s3")) + + testImplementation("${edcGroup}:junit:${edcVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") } java { @@ -28,6 +32,10 @@ java { withSourcesJar() } +tasks.test { + useJUnitPlatform() +} + publishing { publications { create("maven") { diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java index b59afd54..d066b207 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java @@ -14,18 +14,17 @@ package com.ionos.edc.provision.s3; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; import com.ionos.edc.provision.s3.bucket.IonosS3ConsumerResourceDefinitionGenerator; import com.ionos.edc.provision.s3.bucket.IonosS3ProvisionedResource; import com.ionos.edc.provision.s3.bucket.IonosS3Provisioner; import com.ionos.edc.provision.s3.bucket.IonosS3ResourceDefinition; import dev.failsafe.RetryPolicy; -import org.eclipse.edc.connector.transfer.spi.provision.ProvisionManager; -import org.eclipse.edc.connector.transfer.spi.provision.ResourceManifestGenerator; +import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ProvisionManager; +import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; @@ -44,12 +43,9 @@ public class IonosProvisionExtension implements ServiceExtension { @Inject private Vault vault; @Inject - private Monitor monitor; - @Inject private TypeManager typeManager; - @Inject - S3ConnectorApi clientApi; + private S3Connector clientApi; @Override public String name() { @@ -58,7 +54,7 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { - monitor = context.getMonitor(); + var monitor = context.getMonitor(); var keyValidationAttempts = context.getSetting(IONOS_KEY_VALIDATION_ATTEMPTS, IONOS_KEY_VALIDATION_ATTEMPTS_DEFAULT); var keyValidationDelay = context.getSetting(IONOS_KEY_VALIDATION_DELAY, IONOS_KEY_VALIDATION_DELAY_DEFAULT); @@ -67,7 +63,7 @@ public void initialize(ServiceExtensionContext context) { var provisionManager = context.getService(ProvisionManager.class); monitor.debug("IonosProvisionExtension" + "retryPolicy"); - var retryPolicy = (RetryPolicy) context.getService(RetryPolicy.class); + var retryPolicy = context.getService(RetryPolicy.class); monitor.debug("IonosProvisionExtension" + "s3BucketProvisioner"); var s3BucketProvisioner = new IonosS3Provisioner(monitor, retryPolicy, clientApi, keyValidationAttempts, keyValidationDelay); diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java index c27a02c3..4f86d32d 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ConsumerResourceDefinitionGenerator.java @@ -18,22 +18,24 @@ import java.util.Objects; -import org.eclipse.edc.connector.transfer.spi.provision.ConsumerResourceDefinitionGenerator; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ConsumerResourceDefinitionGenerator; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; import org.eclipse.edc.policy.model.Policy; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; import org.eclipse.edc.spi.EdcException; +import org.jetbrains.annotations.Nullable; public class IonosS3ConsumerResourceDefinitionGenerator implements ConsumerResourceDefinitionGenerator { @Override - public ResourceDefinition generate(DataRequest dataRequest, Policy policy) { - Objects.requireNonNull(dataRequest, "dataRequest must always be provided"); + public @Nullable ResourceDefinition generate(TransferProcess transferProcess, Policy policy) { + Objects.requireNonNull(transferProcess, "transferProcess must always be provided"); Objects.requireNonNull(policy, "policy must always be provided"); - var destination = dataRequest.getDataDestination(); + var destination = transferProcess.getDataDestination(); + Objects.requireNonNull(destination, "dataDestination must always be provided"); var path = destination.getStringProperty(IonosBucketSchema.PATH); if ((path != null) && !path.endsWith("/")) { @@ -59,11 +61,11 @@ public ResourceDefinition generate(DataRequest dataRequest, Policy policy) { } @Override - public boolean canGenerate(DataRequest dataRequest, Policy policy) { - Objects.requireNonNull(dataRequest, "dataRequest must always be provided"); + public boolean canGenerate(TransferProcess transferProcess, Policy policy) { + Objects.requireNonNull(transferProcess, "transferProcess must always be provided"); Objects.requireNonNull(policy, "policy must always be provided"); - return IonosBucketSchema.TYPE.equals(dataRequest.getDestinationType()); + return IonosBucketSchema.PUSH_TRANSFER_TYPE.equals(transferProcess.getTransferType()); } } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java index 8b31a33f..b48f0fb1 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.ionos.edc.extension.s3.schema.IonosBucketSchema; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedDataDestinationResource; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedDataDestinationResource; import static com.ionos.edc.extension.s3.schema.IonosBucketSchema.*; diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java index c2f803db..e2149d29 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java @@ -14,16 +14,17 @@ package com.ionos.edc.provision.s3.bucket; -import com.ionos.edc.extension.s3.api.S3ConnectorApi; -import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.S3Connector; +import com.ionos.edc.extension.s3.types.IonosToken; -import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; +import com.ionos.edc.extension.s3.api.S3AccessKey; import dev.failsafe.RetryPolicy; -import org.eclipse.edc.connector.transfer.spi.provision.Provisioner; -import org.eclipse.edc.connector.transfer.spi.types.DeprovisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionResponse; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.connector.controlplane.transfer.spi.provision.Provisioner; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.DeprovisionedResource; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionResponse; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedResource; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.policy.model.Policy; import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.response.StatusResult; @@ -36,15 +37,15 @@ public class IonosS3Provisioner implements Provisioner retryPolicy; - private final S3ConnectorApi s3Api; + private final S3Connector s3Connector; private final Integer keyValidationAttempts; private final Long keyValidationDelay; - public IonosS3Provisioner(Monitor monitor, RetryPolicy retryPolicy, S3ConnectorApi s3Api, int keyValidationAttempts, long keyValidationDelay) { + public IonosS3Provisioner(Monitor monitor, RetryPolicy retryPolicy, S3Connector s3Connector, int keyValidationAttempts, long keyValidationDelay) { this.monitor = monitor; this.retryPolicy = retryPolicy; - this.s3Api = s3Api; + this.s3Connector = s3Connector; this.keyValidationAttempts = keyValidationAttempts; this.keyValidationDelay = keyValidationDelay; } @@ -60,11 +61,10 @@ public boolean canDeprovision(ProvisionedResource resourceDefinition) { } @Override - public CompletableFuture> provision(IonosS3ResourceDefinition resourceDefinition, - org.eclipse.edc.policy.model.Policy policy) { + public CompletableFuture> provision(IonosS3ResourceDefinition resourceDefinition, Policy policy) { String bucketName = resourceDefinition.getBucketName(); - if (!s3Api.bucketExists(bucketName)) { + if (!s3Connector.bucketExists(bucketName)) { createBucket(bucketName); } @@ -98,16 +98,15 @@ public CompletableFuture> provision(IonosS3Resou } @Override - public CompletableFuture> deprovision( - IonosS3ProvisionedResource provisionedResource, org.eclipse.edc.policy.model.Policy policy) { - return with(retryPolicy).runAsync(() -> s3Api.deleteAccessKey(provisionedResource.getAccessKeyID())) + public CompletableFuture> deprovision(IonosS3ProvisionedResource provisionedResource, Policy policy) { + return with(retryPolicy).runAsync(() -> s3Connector.deleteAccessKey(provisionedResource.getAccessKeyID())) .thenApply(empty -> StatusResult.success(DeprovisionedResource.Builder.newInstance().provisionedResourceId(provisionedResource.getId()).build()) ); } private S3AccessKey createTemporaryKey() { - var accessKey = s3Api.createAccessKey(); + var accessKey = s3Connector.createAccessKey(); // Validate the temporary key var validated = false; @@ -125,7 +124,7 @@ private S3AccessKey createTemporaryKey() { return accessKey; } else { // Delete the not validated temporary key - s3Api.deleteAccessKey(accessKey.getId()); + s3Connector.deleteAccessKey(accessKey.getId()); throw new EdcException("Temporary key not validated after " + attempts + " attempts of " + keyValidationDelay + " ms"); } } @@ -139,12 +138,12 @@ private boolean validateKey(S3AccessKey accessKey) { } // Validate the key status - var retrievedAccessKey = s3Api.retrieveAccessKey(accessKey.getId()); + var retrievedAccessKey = s3Connector.retrieveAccessKey(accessKey.getId()); return (retrievedAccessKey.getMetadata().getStatus().equals(S3AccessKey.AVAILABLE_STATUS)); } private void createBucket(String bucketName) { - s3Api.createBucket(bucketName); + s3Connector.createBucket(bucketName); } } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java index 66a3a99c..10b7b8cf 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ResourceDefinition.java @@ -18,10 +18,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; @JsonDeserialize(as=IonosS3ResourceDefinition.class) public class IonosS3ResourceDefinition extends ResourceDefinition { + private String keyName; private String regionId; private String bucketName; diff --git a/extensions/vault-hashicorp/README.md b/extensions/vault-hashicorp/README.md deleted file mode 100644 index 6debcbb3..00000000 --- a/extensions/vault-hashicorp/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# [HashiCorp Vault](https://www.vaultproject.io/) Extension - -## Configuration - -| Key | Description | Mandatory | -|:---|:---|---| -| edc.vault.hashicorp.url | URL to connect to the HashiCorp Vault | X | -| edc.vault.hashicorp.token | Value for [Token Authentication](https://www.vaultproject.io/docs/auth/token) with the vault | X | -| edc.vault.hashicorp.timeout.seconds | Request timeout in seconds when contacting the vault (default: 30) | | - -## Setup vault for integration tests - -The integration tests rely on a vault running locally. -This can be achieved by starting a docker container with the following configuration. - -`docker run -e 'VAULT_DEV_ROOT_TOKEN_ID=test-token' -p "8200:8200" vault:1.9.7` diff --git a/extensions/vault-hashicorp/build.gradle.kts b/extensions/vault-hashicorp/build.gradle.kts deleted file mode 100644 index 798313f0..00000000 --- a/extensions/vault-hashicorp/build.gradle.kts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ -plugins { - `java-library` - `maven-publish` -} - -val edcGroup: String by project -val edcVersion: String by project -val extensionsGroup: String by project -val extensionsVersion: String by project - -val gitHubPkgsName: String by project -val gitHubPkgsUrl: String by project -val gitHubUser: String? by project -val gitHubToken: String? by project - -dependencies { - api("${edcGroup}:core-spi:${edcVersion}") - api("${edcGroup}:http-spi:${edcVersion}") - - implementation("${edcGroup}:util:${edcVersion}") -} - - -java { - withJavadocJar() - withSourcesJar() -} - -publishing { - publications { - create("maven") { - groupId = extensionsGroup - artifactId = "vault-hashicorp" - version = extensionsVersion - - from(components["java"]) - - pom { - name.set("vault-hashicorp") - description.set("Extension to use Hashicorp Vault to store certificates and secrets") - } - } - } - repositories { - maven { - name = gitHubPkgsName - url = uri("https://maven.pkg.github.com/${project.properties["github_owner"]}/${project.properties["github_repo"]}") - credentials { - username = gitHubUser - password = gitHubToken - } - } - } -} \ No newline at end of file diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayload.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayload.java deleted file mode 100644 index aa0e0b9f..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayload.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -import java.util.Map; - -class CreateEntryRequestPayload { - - private CreateEntryRequestPayloadOptions options; - - private Map data; - - CreateEntryRequestPayload() {} - - public CreateEntryRequestPayloadOptions getOptions() { - return this.options; - } - - public Map getData() { - return this.data; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final CreateEntryRequestPayload createEntryRequestPayload; - - private Builder() { - createEntryRequestPayload = new CreateEntryRequestPayload(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder options(CreateEntryRequestPayloadOptions options) { - createEntryRequestPayload.options = options; - return this; - } - - public Builder data(Map data) { - createEntryRequestPayload.data = data; - return this; - } - - public CreateEntryRequestPayload build() { - return createEntryRequestPayload; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayloadOptions.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayloadOptions.java deleted file mode 100644 index 2ac0f69a..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryRequestPayloadOptions.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -class CreateEntryRequestPayloadOptions { - - private Integer cas; - - CreateEntryRequestPayloadOptions() {} - - public Integer getCas() { - return this.cas; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final CreateEntryRequestPayloadOptions createEntryRequestPayloadOptions; - - private Builder() { - createEntryRequestPayloadOptions = new CreateEntryRequestPayloadOptions(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder cas(Integer cas) { - createEntryRequestPayloadOptions.cas = cas; - return this; - } - - public CreateEntryRequestPayloadOptions build() { - return createEntryRequestPayloadOptions; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryResponsePayload.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryResponsePayload.java deleted file mode 100644 index c3187c03..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/CreateEntryResponsePayload.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -class CreateEntryResponsePayload { - - private EntryMetadata data; - - CreateEntryResponsePayload() {} - - public EntryMetadata getData() { - return this.data; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final CreateEntryResponsePayload createEntryResponsePayload; - - private Builder() { - createEntryResponsePayload = new CreateEntryResponsePayload(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder data(EntryMetadata data) { - createEntryResponsePayload.data = data; - return this; - } - - public CreateEntryResponsePayload build() { - return createEntryResponsePayload; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/EntryMetadata.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/EntryMetadata.java deleted file mode 100644 index 9f743cfd..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/EntryMetadata.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -import java.util.Map; - -@JsonDeserialize(builder = EntryMetadata.Builder.class) -class EntryMetadata { - - @JsonProperty() - private Map customMetadata; - - @JsonProperty() - private Boolean destroyed; - - @JsonProperty() - private Integer version; - - EntryMetadata() {} - - public Map getCustomMetadata() { - return this.customMetadata; - } - - public Boolean getDestroyed() { - return this.destroyed; - } - - public Integer getVersion() { - return this.version; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final EntryMetadata entryMetadata; - - Builder() { - entryMetadata = new EntryMetadata(); - } - - @JsonCreator - public static Builder newInstance() { - return new Builder(); - } - - public Builder customMetadata(Map customMetadata) { - entryMetadata.customMetadata = customMetadata; - return this; - } - - public Builder destroyed(Boolean destroyed) { - entryMetadata.destroyed = destroyed; - return this; - } - - public Builder version(Integer version) { - entryMetadata.version = version; - return this; - } - - public EntryMetadata build() { - return entryMetadata; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayload.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayload.java deleted file mode 100644 index 57129a4e..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayload.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -class GetEntryResponsePayload { - - private GetEntryResponsePayloadGetVaultEntryData data; - - GetEntryResponsePayload() { - } - - public GetEntryResponsePayloadGetVaultEntryData getData() { - return this.data; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final GetEntryResponsePayload getEntryResponsePayload; - - private Builder() { - getEntryResponsePayload = new GetEntryResponsePayload(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder data(GetEntryResponsePayloadGetVaultEntryData data) { - getEntryResponsePayload.data = data; - return this; - } - - public GetEntryResponsePayload build() { - return getEntryResponsePayload; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayloadGetVaultEntryData.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayloadGetVaultEntryData.java deleted file mode 100644 index 2241a39a..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/GetEntryResponsePayloadGetVaultEntryData.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; - -import java.util.Map; - -class GetEntryResponsePayloadGetVaultEntryData { - - private Map data; - - private EntryMetadata metadata; - - GetEntryResponsePayloadGetVaultEntryData() {} - - public Map getData() { - return this.data; - } - - public EntryMetadata getMetadata() { - return this.metadata; - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final GetEntryResponsePayloadGetVaultEntryData getEntryResponsePayloadGetVaultEntryData; - - private Builder() { - getEntryResponsePayloadGetVaultEntryData = new GetEntryResponsePayloadGetVaultEntryData(); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder data(Map data) { - getEntryResponsePayloadGetVaultEntryData.data = data; - return this; - } - - public Builder metadata(EntryMetadata metadata) { - getEntryResponsePayloadGetVaultEntryData.metadata = metadata; - return this; - } - - public GetEntryResponsePayloadGetVaultEntryData build() { - return getEntryResponsePayloadGetVaultEntryData; - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVault.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVault.java deleted file mode 100644 index 6063d30f..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVault.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.security.Vault; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -/** - * Implements a vault backed by Hashicorp Vault. - */ -class HashicorpVault implements Vault { - - @NotNull - private final HashicorpVaultClient hashicorpVaultClient; - @NotNull - private final Monitor monitor; - - HashicorpVault(@NotNull HashicorpVaultClient hashicorpVaultClient, @NotNull Monitor monitor) { - this.hashicorpVaultClient = hashicorpVaultClient; - this.monitor = monitor; - } - - @Override - public @Nullable String resolveSecret(String key) { - var result = hashicorpVaultClient.getSecretValue(key); - - return result.succeeded() ? result.getContent() : null; - } - - @Override - public Result storeSecret(String key, String value) { - var result = hashicorpVaultClient.setSecret(key, value); - - return result.succeeded() ? Result.success() : Result.failure(result.getFailureMessages()); - } - - @Override - public Result deleteSecret(String key) { - return hashicorpVaultClient.destroySecret(key); - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultClient.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultClient.java deleted file mode 100644 index 514e0020..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultClient.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import okhttp3.Headers; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import org.eclipse.edc.spi.http.EdcHttpClient; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.TypeManager; -import org.jetbrains.annotations.NotNull; - -import java.io.IOException; -import java.net.URI; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Objects; - -public class HashicorpVaultClient { - static final String VAULT_DATA_ENTRY_NAME = "content"; - private static final String VAULT_TOKEN_HEADER = "X-Vault-Token"; - private static final String VAULT_REQUEST_HEADER = "X-Vault-Request"; - private static final MediaType MEDIA_TYPE_APPLICATION_JSON = MediaType.get("application/json"); - private static final String VAULT_API_VERSION = "v1"; - private static final String VAULT_SECRET_PATH = "secret"; - private static final String VAULT_SECRET_DATA_PATH = "data"; - private static final String VAULT_SECRET_METADATA_PATH = "metadata"; - private static final String CALL_UNSUCCESSFUL_ERROR_TEMPLATE = "[Hashicorp Vault] Call unsuccessful: %s"; - private static final int HTTP_CODE_404 = 404; - @NotNull - private final HashicorpVaultConfig hashicorpVaultConfig; - @NotNull - private final EdcHttpClient httpClient; - @NotNull - private final TypeManager typeManager; - - HashicorpVaultClient(@NotNull HashicorpVaultConfig hashicorpVaultConfig, @NotNull EdcHttpClient httpClient, - @NotNull TypeManager typeManager) { - this.hashicorpVaultConfig = hashicorpVaultConfig; - this.httpClient = httpClient; - this.typeManager = typeManager; - } - - public Result getSecretValue(@NotNull String key) { - var requestUri = getSecretUrl(key, VAULT_SECRET_DATA_PATH); - var headers = getHeaders(); - var request = new Request.Builder().url(requestUri).headers(headers).get().build(); - - try (var response = httpClient.execute(request)) { - - if (response.isSuccessful()) { - if (response.code() == HTTP_CODE_404) { - return Result.failure( - String.format(CALL_UNSUCCESSFUL_ERROR_TEMPLATE, "Secret not found")); - } - - var responseBody = response.body(); - if (responseBody == null) { - return Result.failure(String.format(CALL_UNSUCCESSFUL_ERROR_TEMPLATE, "Response body empty")); - } - var payload = typeManager.readValue(responseBody.string(), GetEntryResponsePayload.class); - var value = payload.getData().getData().get(VAULT_DATA_ENTRY_NAME); - - return Result.success(value); - } else { - return Result.failure(String.format(CALL_UNSUCCESSFUL_ERROR_TEMPLATE, response.code())); - } - - } catch (IOException e) { - return Result.failure(e.getMessage()); - } - } - - public Result setSecret( - @NotNull String key, @NotNull String value) { - var requestUri = getSecretUrl(key, VAULT_SECRET_DATA_PATH); - var headers = getHeaders(); - var requestPayload = - CreateEntryRequestPayload.Builder.newInstance() - .data(Collections.singletonMap(VAULT_DATA_ENTRY_NAME, value)) - .build(); - var request = - new Request.Builder() - .url(requestUri) - .headers(headers) - .post(createRequestBody(requestPayload)) - .build(); - - try (var response = httpClient.execute(request)) { - if (response.isSuccessful()) { - var responseBody = Objects.requireNonNull(response.body()).string(); - var responsePayload = - typeManager.readValue(responseBody, CreateEntryResponsePayload.class); - return Result.success(responsePayload); - } else { - return Result.failure(String.format(CALL_UNSUCCESSFUL_ERROR_TEMPLATE, response.code())); - } - } catch (IOException e) { - return Result.failure(e.getMessage()); - } - } - - public Result destroySecret(@NotNull String key) { - var requestUri = getSecretUrl(key, VAULT_SECRET_METADATA_PATH); - var headers = getHeaders(); - var request = new Request.Builder().url(requestUri).headers(headers).delete().build(); - - try (var response = httpClient.execute(request)) { - return response.isSuccessful() || response.code() == HTTP_CODE_404 - ? Result.success() - : Result.failure(String.format(CALL_UNSUCCESSFUL_ERROR_TEMPLATE, response.code())); - } catch (IOException e) { - return Result.failure(e.getMessage()); - } - } - - @NotNull - private Headers getHeaders() { - var headersBuilder = - new Headers.Builder().add(VAULT_REQUEST_HEADER, Boolean.toString(true)); - if (hashicorpVaultConfig.getVaultToken() != null) { - headersBuilder = headersBuilder.add(VAULT_TOKEN_HEADER, hashicorpVaultConfig.getVaultToken()); - } - return headersBuilder.build(); - } - - private String getBaseUrl() { - var baseUrl = hashicorpVaultConfig.getVaultUrl(); - - if (baseUrl.endsWith("/")) { - baseUrl = baseUrl.substring(0, baseUrl.length() - 1); - } - - return baseUrl; - } - - private String getSecretUrl(String key, String entryType) { - var encodedKey = URLEncoder.encode(key, StandardCharsets.UTF_8); - return URI.create( - String.format( - "%s/%s/%s/%s/%s", - getBaseUrl(), VAULT_API_VERSION, VAULT_SECRET_PATH, entryType, encodedKey)) - .toString(); - } - - private RequestBody createRequestBody(Object requestPayload) { - var jsonRepresentation = typeManager.writeValueAsString(requestPayload); - return RequestBody.create(jsonRepresentation, MEDIA_TYPE_APPLICATION_JSON); - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultConfig.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultConfig.java deleted file mode 100644 index 52c58e25..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -class HashicorpVaultConfig { - private final String vaultUrl; - private final String vaultToken; - - HashicorpVaultConfig(String vaultUrl, String vaultToken) { - this.vaultUrl = vaultUrl; - this.vaultToken = vaultToken; - } - - public String getVaultUrl() { - return this.vaultUrl; - } - - public String getVaultToken() { - return this.vaultToken; - } - - public static class Builder { - private String vaultUrl; - private String vaultToken; - - Builder() { - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder vaultUrl(String vaultUrl) { - this.vaultUrl = vaultUrl; - return this; - } - - public Builder vaultToken(String vaultToken) { - this.vaultToken = vaultToken; - return this; - } - - public HashicorpVaultConfig build() { - return new HashicorpVaultConfig(vaultUrl, vaultToken); - } - } -} diff --git a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultExtension.java b/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultExtension.java deleted file mode 100644 index 068ae052..00000000 --- a/extensions/vault-hashicorp/src/main/java/org/eclipse/edc/vault/hashicorp/HashicorpVaultExtension.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial API and Implementation - * - */ - -package org.eclipse.edc.vault.hashicorp; - -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provider; -import org.eclipse.edc.runtime.metamodel.annotation.Provides; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.http.EdcHttpClient; -import org.eclipse.edc.spi.security.CertificateResolver; -import org.eclipse.edc.spi.security.PrivateKeyResolver; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.security.VaultCertificateResolver; -import org.eclipse.edc.spi.security.VaultPrivateKeyResolver; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.TypeManager; - -@Provides({ Vault.class, PrivateKeyResolver.class, CertificateResolver.class }) -@Extension(value = HashicorpVaultExtension.NAME) -public class HashicorpVaultExtension implements ServiceExtension { - - @Setting(value = "The URL of the Hashicorp Vault", required = true) - public static final String VAULT_URL = "edc.vault.hashicorp.url"; - - @Setting(value = "The token used to access the Hashicorp Vault", required = true) - public static final String VAULT_TOKEN = "edc.vault.hashicorp.token"; - public static final String NAME = "Hashicorp Vault"; - - @Inject - private EdcHttpClient httpClient; - - private Vault vault; - - private PrivateKeyResolver privateKeyResolver; - - @Inject - private TypeManager typeManager; - - @Override - public String name() { - return NAME; - } - - @Provider - public Vault vault() { - return vault; - } - - // @Provider - // public PrivateKeyResolver privateKeyResolver() { - // return privateKeyResolver; - // } - - @Override - public void initialize(ServiceExtensionContext context) { - var config = loadHashicorpVaultClientConfig(context); - - var client = new HashicorpVaultClient(config, httpClient, typeManager); - - vault = new HashicorpVault(client, context.getMonitor()); - privateKeyResolver = new VaultPrivateKeyResolver(vault); - - // context.registerService(CertificateResolver.class, new VaultCertificateResolver(vault)); - } - - private HashicorpVaultConfig loadHashicorpVaultClientConfig( - ServiceExtensionContext context) { - - var vaultUrl = context.getSetting(VAULT_URL, null); - if (vaultUrl == null) { - throw new EdcException(String.format("Vault URL (%s) must be defined", VAULT_URL)); - } - - var vaultToken = context.getSetting(VAULT_TOKEN, null); - - if (vaultToken == null) { - throw new EdcException( - String.format("For Vault authentication [%s] is required", VAULT_TOKEN)); - } - - return HashicorpVaultConfig.Builder.newInstance() - .vaultUrl(vaultUrl) - .vaultToken(vaultToken) - .build(); - } -} diff --git a/extensions/vault-hashicorp/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/vault-hashicorp/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index d6f1ee27..00000000 --- a/extensions/vault-hashicorp/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2022 Mercedes-Benz Tech Innovation GmbH -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Mercedes-Benz Tech Innovation GmbH - Initial ServiceExtension file -# -org.eclipse.edc.vault.hashicorp.HashicorpVaultExtension diff --git a/gradle.properties b/gradle.properties index 103f5177..ef368926 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,17 +1,16 @@ javaVersion=17 extensionsGroup=com.ionoscloud.edc -extensionsVersion=1.0.0 -postgresqlGroup=org.postgresql -postgresqlVersion=42.6.0 +extensionsVersion=2.3.0-SNAPSHOT + edcGroup=org.eclipse.edc -edcVersion=0.4.1 -metaModelVersion=0.0.1-SNAPSHOT -postgresVersion=42.6.0 -rsApi=3.1.0 -minIOVersion=8.5.8 -junitVersion=5.9.1 -mockitoVersion=5.2.0 -ionosDevelopersName=Paulo Lory, Paulo Cabrita +edcVersion=0.7.2 +metaModelVersion=0.7.2 + +minIOVersion=8.5.12 +junitVersion=5.11.0 +mockitoVersion=5.11.0 +postgresVersion=42.7.4 + gitHubPkgsName=GitHubPackages gitHubRpName=edc-ionos-s3 gitHubPkgsUrl=https://maven.pkg.github.com/Digital-Ecosystems/edc-ionos-s3 diff --git a/hashicorp/README.md b/hashicorp/README.md index 2de75207..d6a12a8f 100644 --- a/hashicorp/README.md +++ b/hashicorp/README.md @@ -64,7 +64,7 @@ vault login token= vault kv put secret/edc.ionos.access.key content= vault kv put secret/edc.ionos.secret.key content= vault kv put secret/edc.ionos.token content= -vault kv put secret/edc.ionos.endpoint.region content= +vault kv put secret/edc.ionos.endpoint.region content= ``` Note: @@ -76,10 +76,29 @@ Note: ```console kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.access.key content= kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.secret.key content= -kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content= +kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content= kubectl exec -it vault-0 -- vault kv put secret/edc.ionos.token content= ``` Note: - the `edc.ionos.token` field is only required when you want to do some provisiong tasks; - the `Root Token` of the Hashicorp vault instance is displayed when you start the service; + +## Pull Transfers Support + +If you need to perform pull transfers on your connector, you need to perform the following steps: + +### Using vault CLI +```bash +export VAULT_ADDR= +vault login token= + +vault kv put secret/edc.connector.private.key content=@./certs/private.pem +vault kv put secret/edc.connector.public.key content=@./certs/public.pem +``` + +### Using kubectl +```bash +kubectl exec -it "vault-0" -- vault kv put secret/edc.connector.private.key content="$(cat ./certs/private.pem)" +kubectl exec -it "vault-0" -- vault kv put secret/edc.connector.public.key content="$(cat ./certs/public.pem)" +``` diff --git a/hashicorp/certs/private.pem b/hashicorp/certs/private.pem new file mode 100644 index 00000000..81c28bac --- /dev/null +++ b/hashicorp/certs/private.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIARDUGJgKy1yzxkueIJ1k3MPUWQ/tbQWQNqW6TjyHpdcoAoGCCqGSM49 +AwEHoUQDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5roYnkAXuqCYfNK3ex+hMWFuiX +GUxHlzShAehR6wvwzV23bbC0tcFcVgW//A== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/hashicorp/certs/public.pem b/hashicorp/certs/public.pem new file mode 100644 index 00000000..977a1957 --- /dev/null +++ b/hashicorp/certs/public.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5r +oYnkAXuqCYfNK3ex+hMWFuiXGUxHlzShAehR6wvwzV23bbC0tcFcVgW//A== +-----END PUBLIC KEY----- \ No newline at end of file diff --git a/hashicorp/docker-compose.yml b/hashicorp/docker-compose.yml index aa1f0e0f..41b61b83 100644 --- a/hashicorp/docker-compose.yml +++ b/hashicorp/docker-compose.yml @@ -1,9 +1,7 @@ -version: "3.9" - services: hashicorp-vault: container_name: hashicorp-vault - image: vault:1.9.7 + image: hashicorp/vault:1.17.2 ports: - "8200:8200" environment: diff --git a/launchers/base/connector/build.gradle.kts b/launchers/base/connector/build.gradle.kts index ce111b09..fa5c33ac 100644 --- a/launchers/base/connector/build.gradle.kts +++ b/launchers/base/connector/build.gradle.kts @@ -16,41 +16,40 @@ plugins { `java-library` } -repositories { - maven {// while runtime-metamodel dependency is still a snapshot - url = uri("https://oss.sonatype.org/content/repositories/snapshots/") - } - mavenCentral() - mavenLocal() -} - val edcGroup: String by project val edcVersion: String by project dependencies { - implementation("${edcGroup}:boot:${edcVersion}") - - // Control Plane - implementation("${edcGroup}:control-plane-core:${edcVersion}") - implementation("${edcGroup}:control-plane-api:${edcVersion}") - implementation("${edcGroup}:control-plane-api-client:${edcVersion}") - + // Core + implementation("${edcGroup}:connector-core:${edcVersion}") implementation("${edcGroup}:http:${edcVersion}") implementation("${edcGroup}:dsp:${edcVersion}") - implementation("${edcGroup}:auth-tokenbased:${edcVersion}") - implementation("${edcGroup}:configuration-filesystem:${edcVersion}") - - implementation("$edcGroup:management-api:$edcVersion") + implementation("${edcGroup}:management-api:${edcVersion}") implementation("${edcGroup}:api-observability:${edcVersion}") - implementation("${edcGroup}:vault-hashicorp:${edcVersion}") - + // Control Plane + implementation("${edcGroup}:control-plane-api-client:${edcVersion}") + implementation("${edcGroup}:control-plane-api:${edcVersion}") + implementation("${edcGroup}:control-plane-core:${edcVersion}") + implementation("${edcGroup}:control-api-configuration:${edcVersion}") + // Data Plane - implementation("${edcGroup}:data-plane-selector-client:${edcVersion}") + implementation("${edcGroup}:data-plane-selector-api:${edcVersion}") implementation("${edcGroup}:data-plane-selector-core:${edcVersion}") + implementation("${edcGroup}:data-plane-self-registration:${edcVersion}") + implementation("${edcGroup}:data-plane-control-api:${edcVersion}") + implementation("${edcGroup}:data-plane-public-api-v2:${edcVersion}") implementation("${edcGroup}:data-plane-core:${edcVersion}") - implementation("${edcGroup}:data-plane-client:${edcVersion}") - implementation("${edcGroup}:transfer-data-plane:${edcVersion}") + implementation("${edcGroup}:data-plane-http:${edcVersion}") + implementation("${edcGroup}:transfer-data-plane-signaling:${edcVersion}") + + // EDR Cache + implementation("${edcGroup}:edr-cache-api:${edcVersion}") + implementation("${edcGroup}:edr-store-core:${edcVersion}") + implementation("${edcGroup}:edr-store-receiver:${edcVersion}") + + // Validators + implementation("${edcGroup}:validator-data-address-http-data:${edcVersion}") // Ionos Extensions implementation(project(":extensions:provision-ionos-s3")) diff --git a/launchers/dev/connector-consumer/build.gradle.kts b/launchers/dev/connector-consumer/build.gradle.kts index ec655d34..4b34d51c 100644 --- a/launchers/dev/connector-consumer/build.gradle.kts +++ b/launchers/dev/connector-consumer/build.gradle.kts @@ -24,14 +24,11 @@ val edcVersion: String by project dependencies { implementation(project(":launchers:base:connector")) + implementation("${edcGroup}:configuration-filesystem:${edcVersion}") + implementation("${edcGroup}:vault-hashicorp:${edcVersion}") implementation("${edcGroup}:iam-mock:${edcVersion}") } application { mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") } - -java { - -} - diff --git a/launchers/dev/connector-consumer/resources/config.properties b/launchers/dev/connector-consumer/resources/config.properties index 63d85447..e6246173 100644 --- a/launchers/dev/connector-consumer/resources/config.properties +++ b/launchers/dev/connector-consumer/resources/config.properties @@ -18,5 +18,5 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= -edc.ionos.endpoint.region= -edc.ionos.token= \ No newline at end of file +edc.ionos.endpoint=https://s3-eu-central-1.ionoscloud.com +edc.ionos.token= \ No newline at end of file diff --git a/launchers/dev/connector-provider/build.gradle.kts b/launchers/dev/connector-provider/build.gradle.kts index 78106416..4319e3cb 100644 --- a/launchers/dev/connector-provider/build.gradle.kts +++ b/launchers/dev/connector-provider/build.gradle.kts @@ -24,6 +24,8 @@ val edcVersion: String by project dependencies { implementation(project(":launchers:base:connector")) + implementation("${edcGroup}:configuration-filesystem:${edcVersion}") + implementation("${edcGroup}:vault-hashicorp:${edcVersion}") implementation("${edcGroup}:iam-mock:${edcVersion}") } diff --git a/launchers/dev/connector-provider/resources/config.properties b/launchers/dev/connector-provider/resources/config.properties index 3717bddc..9e5211c7 100644 --- a/launchers/dev/connector-provider/resources/config.properties +++ b/launchers/dev/connector-provider/resources/config.properties @@ -1,5 +1,4 @@ edc.participant.id=provider -edc.dsp.callback.address=http://localhost:8282/protocol web.http.port=8181 web.http.path=/api web.http.management.port=8182 @@ -10,7 +9,13 @@ web.http.control.port=8283 web.http.control.path=/control web.http.public.port=8383 web.http.public.path=/public +edc.dsp.callback.address=http://localhost:8282/protocol +edc.dataplane.token.validation.endpoint=http://localhost:8283/control/token +edc.dataplane.api.public.baseurl=http://localhost:8383/public + edc.api.auth.key=password +edc.transfer.proxy.token.signer.privatekey.alias=edc.connector.private.key +edc.transfer.proxy.token.verifier.publickey.alias=edc.connector.public.key edc.vault.hashicorp.url=http://localhost:8200 edc.vault.hashicorp.token=test-token @@ -19,4 +24,4 @@ edc.vault.hashicorp.timeout.seconds=30 edc.ionos.access.key= edc.ionos.secret.key= edc.ionos.endpoint.region= -edc.ionos.token= \ No newline at end of file +edc.ionos.token= diff --git a/launchers/prod/connector-persistence/README.md b/launchers/prod/connector-persistence/README.md index 866ce705..14e902dd 100644 --- a/launchers/prod/connector-persistence/README.md +++ b/launchers/prod/connector-persistence/README.md @@ -25,9 +25,17 @@ Just check the `Configuration` section of the example [readme](../example/README Open the `resources/config.properties` file and insert the key and the secret of your IONOS S3 storage and the token. -### Import the initial database +### Create the initial database schemas ``` -psql -h -p -U < ../deployment/terraform/db-scripts/init.sql +psql -h -p -U < ../deployment/terraform/db-scripts/accesstokendata-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/asset-index/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/contract-definition-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/contract-negotiation-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/data-plane-instance-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/data-plane-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/edr-index/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/policy-definition-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/transfer-process-store/schema.sql ``` ## Building and running the docker diff --git a/launchers/prod/connector-persistence/build.gradle.kts b/launchers/prod/connector-persistence/build.gradle.kts index de89a350..0ee4d122 100644 --- a/launchers/prod/connector-persistence/build.gradle.kts +++ b/launchers/prod/connector-persistence/build.gradle.kts @@ -25,18 +25,25 @@ val postgresVersion: String by project dependencies { implementation(project(":launchers:base:connector")) + implementation("${edcGroup}:configuration-filesystem:${edcVersion}") + implementation("${edcGroup}:vault-hashicorp:${edcVersion}") + implementation("${edcGroup}:iam-mock:${edcVersion}") + implementation("org.postgresql:postgresql:$postgresVersion") implementation("${edcGroup}:sql-pool-apache-commons:$edcVersion") implementation("${edcGroup}:transaction-local:$edcVersion") implementation("${edcGroup}:transaction-datasource-spi:$edcVersion") + implementation("${edcGroup}:accesstokendata-store-sql:$edcVersion") implementation("${edcGroup}:asset-index-sql:$edcVersion") - implementation("${edcGroup}:policy-definition-store-sql:$edcVersion") implementation("${edcGroup}:contract-definition-store-sql:$edcVersion") implementation("${edcGroup}:contract-negotiation-store-sql:$edcVersion") + implementation("${edcGroup}:control-plane-sql:$edcVersion") + implementation("${edcGroup}:data-plane-instance-store-sql:$edcVersion") + implementation("${edcGroup}:data-plane-store-sql:$edcVersion") + implementation("${edcGroup}:edr-index-sql:$edcVersion") + implementation("${edcGroup}:policy-definition-store-sql:$edcVersion") implementation("${edcGroup}:transfer-process-store-sql:$edcVersion") - - implementation("${edcGroup}:iam-mock:${edcVersion}") } application { diff --git a/launchers/prod/connector-persistence/docker-compose.yml b/launchers/prod/connector-persistence/docker-compose.yml index e85fa2e9..de030301 100644 --- a/launchers/prod/connector-persistence/docker-compose.yml +++ b/launchers/prod/connector-persistence/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.9" - services: connector: image: edc-ionos-s3 diff --git a/launchers/prod/connector-persistence/resources/config.properties b/launchers/prod/connector-persistence/resources/config.properties index 36d759a6..418b9d39 100644 --- a/launchers/prod/connector-persistence/resources/config.properties +++ b/launchers/prod/connector-persistence/resources/config.properties @@ -1,3 +1,4 @@ +edc.participant.id=connector web.http.port=8181 web.http.path=/api web.http.management.port=8182 @@ -8,12 +9,14 @@ web.http.control.port=8283 web.http.control.path=/control web.http.public.port=8383 web.http.public.path=/public +edc.dsp.callback.address=http://localhost:8282/protocol +edc.dataplane.token.validation.endpoint=http://localhost:8283/control/token +edc.dataplane.api.public.baseurl=http://localhost:8383/public + edc.api.auth.key=password +edc.transfer.proxy.token.signer.privatekey.alias=edc.connector.private.key +edc.transfer.proxy.token.verifier.publickey.alias=edc.connector.public.key -edc.vault.clientid=company1 -edc.vault.tenantid=1 -edc.vault.certificate=/resources/ -edc.vault.name=ionos edc.vault.hashicorp.url=http://localhost:8200 edc.vault.hashicorp.token=test-token edc.vault.hashicorp.timeout.seconds=30 diff --git a/launchers/prod/connector/build.gradle.kts b/launchers/prod/connector/build.gradle.kts index 2fe35970..10ca5508 100644 --- a/launchers/prod/connector/build.gradle.kts +++ b/launchers/prod/connector/build.gradle.kts @@ -24,7 +24,9 @@ val edcVersion: String by project dependencies { implementation(project(":launchers:base:connector")) - implementation("${edcGroup}:iam-mock:${edcVersion}") + implementation("${edcGroup}:configuration-filesystem:${edcVersion}") + implementation("${edcGroup}:vault-hashicorp:${edcVersion}") + implementation("${edcGroup}:iam-mock:${edcVersion}") } application { diff --git a/launchers/prod/connector/docker-compose.yml b/launchers/prod/connector/docker-compose.yml index e85fa2e9..de030301 100644 --- a/launchers/prod/connector/docker-compose.yml +++ b/launchers/prod/connector/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.9" - services: connector: image: edc-ionos-s3 diff --git a/launchers/prod/connector/resources/config.properties b/launchers/prod/connector/resources/config.properties index 7c848bc8..2b1aacd5 100644 --- a/launchers/prod/connector/resources/config.properties +++ b/launchers/prod/connector/resources/config.properties @@ -1,3 +1,4 @@ +edc.participant.id=connector web.http.port=8181 web.http.path=/api web.http.management.port=8182 @@ -8,12 +9,14 @@ web.http.control.port=8283 web.http.control.path=/control web.http.public.port=8383 web.http.public.path=/public +edc.dsp.callback.address=http://localhost:8282/protocol +edc.dataplane.token.validation.endpoint=http://localhost:8283/control/token +edc.dataplane.api.public.baseurl=http://localhost:8383/public + edc.api.auth.key=password +edc.transfer.proxy.token.signer.privatekey.alias=edc.connector.private.key +edc.transfer.proxy.token.verifier.publickey.alias=edc.connector.public.key -edc.vault.clientid=company1 -edc.vault.tenantid=1 -edc.vault.certificate=/resources/ -edc.vault.name=ionos edc.vault.hashicorp.url=http://localhost:8200 edc.vault.hashicorp.token=test-token edc.vault.hashicorp.timeout.seconds=30 diff --git a/settings.gradle.kts b/settings.gradle.kts index 2fcb8960..3e5448be 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,13 +10,14 @@ dependencyResolutionManagement { repositories { - mavenCentral() mavenLocal() } versionCatalogs { + val group = providers.gradleProperty("edcGroup") + val version = providers.gradleProperty("edcVersion") create("libs") { - from("org.eclipse.edc:edc-versions:0.1.2") + from(group.get() + ":edc-versions:" + version.get()) } } } @@ -24,7 +25,6 @@ dependencyResolutionManagement { include(":extensions:data-plane-ionos-s3") include(":extensions:provision-ionos-s3") include(":extensions:core-ionos-s3") -include(":extensions:vault-hashicorp") include(":launchers:base:connector") include(":launchers:dev:connector-consumer")