Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add persistence for data-plane-selector #1475

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
422 changes: 212 additions & 210 deletions DEPENDENCIES

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,16 @@ spec:
- name: "EDC_DATASOURCE_FEDERATEDCATALOG_URL"
value: {{ tpl .Values.postgresql.jdbcUrl . | quote }}

# see extension https://github.com/eclipse-edc/Connector/tree/main/extensions/data-plane-selector/store/sql/data-plane-instance-store-sql
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_NAME"
value: "dataplaneinstance"
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_USER"
value: {{ .Values.postgresql.auth.username | required ".Values.postgresql.auth.username is required" | quote }}
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_PASSWORD"
value: {{ .Values.postgresql.auth.password | required ".Values.postgresql.auth.password is required" | quote }}
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_URL"
value: {{ tpl .Values.postgresql.jdbcUrl . | quote }}

#############################
## IATP / STS / DIM CONFIG ##
#############################
Expand Down
13 changes: 12 additions & 1 deletion charts/tractusx-connector/templates/deployment-controlplane.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,18 @@ spec:
- name: "EDC_DATASOURCE_FEDERATEDCATALOG_URL"
value: {{ tpl .Values.postgresql.jdbcUrl . | quote }}

#############################
# see extension https://github.com/eclipse-edc/Connector/tree/main/extensions/data-plane-selector/store/sql/data-plane-instance-store-sql
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_NAME"
value: "dataplaneinstance"
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_USER"
value: {{ .Values.postgresql.auth.username | required ".Values.postgresql.auth.username is required" | quote }}
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_PASSWORD"
value: {{ .Values.postgresql.auth.password | required ".Values.postgresql.auth.password is required" | quote }}
- name: "EDC_DATASOURCE_DATAPLANEINSTANCE_URL"
value: {{ tpl .Values.postgresql.jdbcUrl . | quote }}


#############################
## IATP / STS / DIM CONFIG ##
#############################
- name: "EDC_IAM_STS_OAUTH_TOKEN_URL"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/********************************************************************************
* Copyright (c) 2024 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
********************************************************************************/

package org.eclipse.tractusx.edc.postgresql.migration;

public class DataPlaneInstancePostgresqlMigrationExtension extends AbstractPostgresqlMigrationExtension {
private static final String NAME_SUBSYSTEM = "dataplaneinstance";

protected String getSubsystemName() {

Check notice

Code scanning / CodeQL

Missing Override annotation Note

This method overrides
AbstractPostgresqlMigrationExtension.getSubsystemName
; it is advisable to add an Override annotation.
return NAME_SUBSYSTEM;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@


org.eclipse.tractusx.edc.postgresql.migration.AssetPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.BusinessGroupPostgresMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.ContractDefinitionPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.ContractNegotiationPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.PolicyPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.DataPlaneInstancePostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.EdrIndexPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.FederatedCatalogCacheMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.PolicyMonitorPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.PolicyPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.TransferProcessPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.BusinessGroupPostgresMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.EdrIndexPostgresqlMigrationExtension
org.eclipse.tractusx.edc.postgresql.migration.FederatedCatalogCacheMigrationExtension
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
--
-- Copyright (c) 2024 Contributors to the Eclipse Foundation
--
-- This program and the accompanying materials are made available under the
-- terms of the Apache License, Version 2.0 which is available at
-- https://www.apache.org/licenses/LICENSE-2.0
--
-- SPDX-License-Identifier: Apache-2.0
--
-- Contributors:
-- Contributors to the Eclipse Foundation - initial API and implementation
--


CREATE TABLE IF NOT EXISTS edc_lease
(
leased_by VARCHAR NOT NULL,
leased_at BIGINT,
lease_duration INTEGER NOT NULL,
lease_id VARCHAR NOT NULL
CONSTRAINT lease_pk
PRIMARY KEY
);


CREATE TABLE IF NOT EXISTS edc_data_plane_instance
(
id VARCHAR NOT NULL PRIMARY KEY,
data JSON,
lease_id VARCHAR
CONSTRAINT data_plane_instance_lease_id_fk
REFERENCES edc_lease
ON DELETE SET NULL
);
Comment on lines +15 to +34
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do the schemas of these tables match the old tables, so that in case of an upgrade of the edc (even if it is not supported) there should be no problems?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you mean with "match the old tables"?
in case of upgrade (that AFAIR should be actually possible) edc_lease won't be created as already existent and edc_data_plane_instance will be created.

Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class PgRuntimeExtension extends ParticipantRuntimeExtension {
private static final String PASSWORD = "password";
private static final List<String> DATASOURCES = List.of("asset", "contractdefinition",
"contractnegotiation", "policy", "transferprocess", "bpn",
"policy-monitor", "edr", "dataplane", "accesstokendata", "federatedcatalog");
"policy-monitor", "edr", "dataplane", "accesstokendata", "federatedcatalog", "dataplaneinstance");
private final PostgreSQLContainer<?> postgreSqlContainer;
private final String dbName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ dependencies {
testImplementation(libs.edc.config.filesystem)
testImplementation(libs.edc.dpf.http)
testImplementation(libs.edc.auth.tokenbased)
testImplementation(libs.edc.dpf.selector.spi)
testImplementation(libs.edc.spi.dataplane.selector)
testImplementation(libs.testcontainers.junit)
testImplementation(libs.edc.aws.s3.core)
testImplementation(libs.aws.s3)
Expand Down
24 changes: 15 additions & 9 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
format.version = "1.1"

[versions]
edc = "0.8.1"
fc = "0.8.1"
edc = "0.8.2-20240801-SNAPSHOT"

assertj = "3.26.3"
awaitility = "4.2.1"
aws = "2.26.25"
Expand Down Expand Up @@ -162,7 +162,7 @@ edc-identity-trust-sts-client-configuration = { module = "org.eclipse.edc:identi
edc-fc-spi-crawler = { module = "org.eclipse.edc:crawler-spi", version.ref = "edc" }
edc-fc-core = { module = "org.eclipse.edc:federated-catalog-core", version.ref = "edc" }
edc-fc-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref = "edc" }
edc-fc-sql-cache = { module = "org.eclipse.edc:federated-catalog-cache-sql", version.ref = "fc" }
edc-fc-sql-cache = { module = "org.eclipse.edc:federated-catalog-cache-sql", version.ref = "edc" }

## IH for testing

Expand All @@ -176,14 +176,18 @@ edc-ih-participants = { module = "org.eclipse.edc:identity-hub-participants", ve
edc-ih-did = { module = "org.eclipse.edc:identity-hub-did", version.ref = "edc" }
edc-ih-core = { module = "org.eclipse.edc:identity-hub-core", version.ref = "edc" }

# Data plane selector
edc-spi-dataplane-selector = { module = "org.eclipse.edc:data-plane-selector-spi", version.ref = "edc" }
edc-dpf-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" }
edc-data-plane-selector-client = { module = "org.eclipse.edc:data-plane-selector-client", version.ref = "edc" }
edc-data-plane-selector-control-api = { module = "org.eclipse.edc:data-plane-selector-control-api", version.ref = "edc" }
edc-data-plane-instance-store-sql = { module = "org.eclipse.edc:data-plane-instance-store-sql", version.ref = "edc" }

# DPF modules
edc-spi-dataplane-dataplane = { module = "org.eclipse.edc:data-plane-spi", version.ref = "edc" }
edc-spi-dataplane-transfer = { module = "org.eclipse.edc:transfer-data-plane-spi", version.ref = "edc" }
edc-spi-dataplane-selector = { module = "org.eclipse.edc:data-plane-selector-spi", version.ref = "edc" }
edc-spi-dataplane-http = { module = "org.eclipse.edc:data-plane-http-spi", version.ref = "edc" }
edc-dpf-transferclient = { module = "org.eclipse.edc:data-plane-transfer-client", version.ref = "edc" }
edc-dpf-selector-spi = { module = "org.eclipse.edc:data-plane-selector-spi", version.ref = "edc" }
edc-dpf-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" }
edc-dpf-transfer-signaling = { module = "org.eclipse.edc:transfer-data-plane-signaling", version.ref = "edc" }
edc-dpf-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" }
edc-dpf-util = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" }
Expand All @@ -194,8 +198,6 @@ edc-dpf-api-control = { module = "org.eclipse.edc:data-plane-control-api", versi
edc-dpf-api-public-v2 = { module = "org.eclipse.edc:data-plane-public-api-v2", version.ref = "edc" }

edc-dpf-api-signaling = { module = "org.eclipse.edc:data-plane-signaling-api", version.ref = "edc" }
edc-data-plane-selector-control-api = { module = "org.eclipse.edc:data-plane-selector-control-api", version.ref = "edc" }
edc-data-plane-selector-client = { module = "org.eclipse.edc:data-plane-selector-client", version.ref = "edc" }
edc-data-plane-self-registration = { module = "org.eclipse.edc:data-plane-self-registration", version.ref = "edc" }

# micrometer and other infra stuff
Expand Down Expand Up @@ -237,7 +239,11 @@ titaniumJsonLd = { module = "com.apicatalog:titanium-json-ld", version.ref = "ti
[bundles]
edc-connector = ["edc.boot", "edc.core-connector", "edc.core-controlplane", "edc.api-observability"]
edc-dpf = ["edc.dpf-transfer-signaling", "edc.dpf-selector-core", "edc.spi-dataplane-selector"]
edc-sqlstores = ["edc.sql-assetindex", "edc.sql-contract-definition", "edc.sql-contract-negotiation", "edc.sql-transferprocess", "edc.sql-policydef", "edc.sql-policy-monitor", "edc.sql-edrindex", "edc-transaction-local", "edc-fc-sql-cache"]
edc-sqlstores = [
"edc-sql-assetindex", "edc-sql-contract-definition", "edc-sql-contract-negotiation", "edc-sql-transferprocess",
"edc-sql-policydef", "edc-sql-policy-monitor", "edc-sql-edrindex", "edc-transaction-local", "edc-fc-sql-cache",
"edc-data-plane-instance-store-sql"
]
edc-monitoring = ["edc.micrometer-core", "edc.micrometer-jersey", "edc.micrometer-jetty"]
edc-sts = ["edc-identity-trust-sts-core", "edc-identity-trust-sts-api", "edc-identity-trust-sts-client-configuration"]

Expand Down
Loading