feature: added postgres support

YarikRevich · Dec 5, 2024 · 2c01021 · 2c01021
1 parent 0687917
commit 2c01021
Show file tree

Hide file tree

Showing 24 changed files with 420 additions and 175 deletions.
diff --git a/README.md b/README.md
@@ -144,6 +144,9 @@ diagnostics:
     port: 8121
 ```
 
+In the **~/.objectstorage/internal/database** directory there will be located internal database data, if **sqlite3** 
+option is selected as target database.
+
 ### Diagnostics dashboard
 
 For **ObjectStorage API Server** configuration the following section should be modified:
diff --git a/api-server/pom.xml b/api-server/pom.xml
@@ -100,6 +100,10 @@
             <groupId>io.quarkiverse.jdbc</groupId>
             <artifactId>quarkus-jdbc-sqlite</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-jdbc-postgresql</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-smallrye-jwt</artifactId>

diff --git a/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java
@@ -51,6 +51,55 @@ public static class Security {
     @JsonProperty("connection")
     public Connection connection;
 
+    /**
+     * Represents ObjectStorage internal storage configuration used for internal database setup.
+     */
+    @Getter
+    public static class InternalStorage {
+        /**
+         * Represents all supported providers, which can be used by ObjectStorage internal storage.
+         */
+        @Getter
+        public enum Provider {
+            @JsonProperty("sqlite3")
+            SQLITE3("sqlite3"),
+
+            @JsonProperty("postgres")
+            POSTGRES("postgres");
+
+            private final String value;
+
+            Provider(String value) {
+                this.value = value;
+            }
+
+            public String toString() {
+                return value;
+            }
+        }
+
+        @Valid
+        @NotNull
+        @JsonProperty("provider")
+        public Provider provider;
+
+        @JsonProperty("host")
+        public String host;
+
+        @NotNull
+        @JsonProperty("username")
+        public String username;
+
+        @NotNull
+        @JsonProperty("password")
+        public String password;
+    }
+
+    @Valid
+    @NotNull
+    @JsonProperty("internal-storage")
+    public InternalStorage internalStorage;
+
     /**
      * Represents ObjectStorage API Server configuration used for temporate storage setup.
      */

diff --git a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java
@@ -20,6 +20,15 @@ public class PropertiesEntity {
     @ConfigProperty(name = "quarkus.http.port")
     Integer applicationPort;
 
+    @ConfigProperty(name = "database.name")
+    String databaseName;
+
+    @ConfigProperty(name = "liquibase.sqlite3.config")
+    String liquibaseSqlite3Config;
+
+    @ConfigProperty(name = "liquibase.postgres.config")
+    String liquibasePostgresConfig;
+
     @ConfigProperty(name = "content.root.notation")
     String contentRootNotation;
 

diff --git a/...r/src/main/java/com/objectstorage/exception/ConfigDatabasePropertiesMissingException.java b/...r/src/main/java/com/objectstorage/exception/ConfigDatabasePropertiesMissingException.java
@@ -0,0 +1,21 @@
+package com.objectstorage.exception;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Formatter;
+
+/**
+ * Represents exception used when configuration file database properties are missing.
+ */
+public class ConfigDatabasePropertiesMissingException extends IOException {
+    public ConfigDatabasePropertiesMissingException() {
+        this("");
+    }
+
+    public ConfigDatabasePropertiesMissingException(Object... message) {
+        super(
+                new Formatter()
+                        .format("Config file database properties are missing: %s", Arrays.stream(message).toArray())
+                        .toString());
+    }
+}
diff --git a/api-server/src/main/java/com/objectstorage/repository/executor/RepositoryExecutor.java b/api-server/src/main/java/com/objectstorage/repository/executor/RepositoryExecutor.java
@@ -1,8 +1,10 @@
 package com.objectstorage.repository.executor;
 
+import com.objectstorage.entity.common.ConfigEntity;
 import com.objectstorage.entity.common.PropertiesEntity;
 import com.objectstorage.exception.*;
 import com.objectstorage.repository.common.RepositoryConfigurationHelper;
+import com.objectstorage.service.config.ConfigService;
 import jakarta.annotation.PostConstruct;
 import jakarta.annotation.PreDestroy;
 import jakarta.enterprise.context.ApplicationScoped;
@@ -28,6 +30,9 @@ public class RepositoryExecutor {
     @Inject
     PropertiesEntity properties;
 
+    @Inject
+    ConfigService configService;
+
     @Inject
     DataSource dataSource;
 
@@ -53,6 +58,16 @@ private void configure() throws QueryExecutionFailureException {
         } catch (SQLException e) {
             throw new QueryExecutionFailureException(e.getMessage());
         }
+
+        if (configService.getConfig().getInternalStorage().getProvider() ==
+                ConfigEntity.InternalStorage.Provider.POSTGRES) {
+
+            try {
+                performQuery(String.format("CREATE DATABASE IF NOT EXISTS %s", properties.getDatabaseName()));
+            } catch (QueryEmptyResultException e) {
+                throw new QueryExecutionFailureException(e.getMessage());
+            }
+        }
     }
 
     /**

diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java
@@ -85,8 +85,6 @@ public void process() throws BackupPeriodRetrievalFailureException {
                 } catch (ContentApplicationRetrievalFailureException e) {
                     StateService.getBackupProcessorGuard().unlock();
 
-                    logger.error(e.getMessage());
-
                     return;
                 }
 

diff --git a/...main/java/com/objectstorage/service/integration/diagnostics/DiagnosticsConfigService.java b/...main/java/com/objectstorage/service/integration/diagnostics/DiagnosticsConfigService.java
@@ -159,22 +159,12 @@ private void process() throws
                             configService.getConfig().getDiagnostics().getNodeExporter().getPort(),
                             properties.getDiagnosticsCommonDockerNetworkName());
 
-            CommandExecutorOutputDto nodeExporterDeployCommandOutput;
-
             try {
-                nodeExporterDeployCommandOutput =
-                        commandExecutorService.executeCommand(nodeExporterDeployCommandService);
+                commandExecutorService.executeCommand(nodeExporterDeployCommandService);
             } catch (CommandExecutorException e) {
                 throw new NodeExporterDeploymentFailureException(e.getMessage());
             }
 
-            String nodeExporterDeployCommandErrorOutput = nodeExporterDeployCommandOutput.getErrorOutput();
-
-            if (Objects.nonNull(nodeExporterDeployCommandErrorOutput) &&
-                    !nodeExporterDeployCommandErrorOutput.isEmpty()) {
-                throw new NodeExporterDeploymentFailureException(nodeExporterDeployCommandErrorOutput);
-            }
-
             PrometheusDeployCommandService prometheusDeployCommandService =
                     new PrometheusDeployCommandService(
                             properties.getDiagnosticsPrometheusDockerName(),
@@ -184,22 +174,12 @@ private void process() throws
                             properties.getDiagnosticsPrometheusConfigLocation(),
                             properties.getDiagnosticsPrometheusInternalLocation());
 
-            CommandExecutorOutputDto prometheusDeployCommandOutput;
-
             try {
-                prometheusDeployCommandOutput =
-                        commandExecutorService.executeCommand(prometheusDeployCommandService);
+                 commandExecutorService.executeCommand(prometheusDeployCommandService);
             } catch (CommandExecutorException e) {
                 throw new PrometheusDeploymentFailureException(e.getMessage());
             }
 
-            String prometheusDeployCommandErrorOutput = prometheusDeployCommandOutput.getErrorOutput();
-
-            if (Objects.nonNull(prometheusDeployCommandErrorOutput) &&
-                    !prometheusDeployCommandErrorOutput.isEmpty()) {
-                throw new PrometheusDeploymentFailureException(prometheusDeployCommandErrorOutput);
-            }
-
             GrafanaDeployCommandService grafanaDeployCommandService =
                     new GrafanaDeployCommandService(
                             properties.getDiagnosticsGrafanaDockerName(),
@@ -209,21 +189,11 @@ private void process() throws
                             properties.getDiagnosticsGrafanaConfigLocation(),
                             properties.getDiagnosticsGrafanaInternalLocation());
 
-            CommandExecutorOutputDto grafanaDeployCommandOutput;
-
             try {
-                grafanaDeployCommandOutput =
-                        commandExecutorService.executeCommand(grafanaDeployCommandService);
+                commandExecutorService.executeCommand(grafanaDeployCommandService);
             } catch (CommandExecutorException e) {
                 throw new GrafanaDeploymentFailureException(e.getMessage());
             }
-
-            String grafanaDeployCommandErrorOutput = grafanaDeployCommandOutput.getErrorOutput();
-
-            if (Objects.nonNull(grafanaDeployCommandErrorOutput) &&
-                    !grafanaDeployCommandErrorOutput.isEmpty()) {
-                throw new GrafanaDeploymentFailureException(grafanaDeployCommandErrorOutput);
-            }
         }
     }
 

diff --git a/...bjectstorage/service/integration/properties/database/DatabasePropertiesConfigService.java b/...bjectstorage/service/integration/properties/database/DatabasePropertiesConfigService.java
@@ -0,0 +1,102 @@
+package com.objectstorage.service.integration.properties.database;
+
+import com.objectstorage.exception.ConfigDatabasePropertiesMissingException;
+import com.objectstorage.service.config.common.ConfigConfigurationHelper;
+import io.quarkus.runtime.annotations.StaticInitSafe;
+import io.smallrye.config.ConfigSourceContext;
+import io.smallrye.config.ConfigSourceFactory;
+import io.smallrye.config.ConfigValue;
+import io.smallrye.config.PropertiesConfigSource;
+import lombok.SneakyThrows;
+import org.eclipse.microprofile.config.spi.ConfigSource;
+import com.objectstorage.entity.common.ConfigEntity;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.OptionalInt;
+import java.util.Properties;
+
+/**
+ * Service used to perform security properties configuration operations.
+ */
+@StaticInitSafe
+public class DatabasePropertiesConfigService implements ConfigSourceFactory {
+    @Override
+    @SneakyThrows
+    public Iterable<ConfigSource> getConfigSources(final ConfigSourceContext context) {
+        final ConfigValue configLocation = context.getValue("config.location");
+
+        if (Objects.isNull(configLocation) || Objects.isNull(configLocation.getValue())) {
+            return Collections.emptyList();
+        }
+
+        final ConfigValue databaseName = context.getValue("database.name");
+        if (Objects.isNull(databaseName) || Objects.isNull(databaseName.getValue())) {
+            return Collections.emptyList();
+        }
+
+        final ConfigValue liquibaseSqlite3Config = context.getValue("liquibase.sqlite3.config");
+        if (Objects.isNull(liquibaseSqlite3Config) || Objects.isNull(liquibaseSqlite3Config.getValue())) {
+            return Collections.emptyList();
+        }
+
+        final ConfigValue liquibasePostgresConfig = context.getValue("liquibase.postgres.config");
+        if (Objects.isNull(liquibasePostgresConfig) || Objects.isNull(liquibasePostgresConfig.getValue())) {
+            return Collections.emptyList();
+        }
+
+        Properties properties = new Properties();
+
+        ConfigEntity config = ConfigConfigurationHelper.readConfig(configLocation.getValue(), false);
+        if (Objects.isNull(config)) {
+            return Collections.emptyList();
+        }
+
+        if (Objects.isNull(config.getInternalStorage()) ||
+                Objects.isNull(config.getInternalStorage().getProvider()) ||
+                Objects.isNull(config.getInternalStorage().getUsername()) ||
+                Objects.isNull(config.getInternalStorage().getPassword())) {
+            throw new ConfigDatabasePropertiesMissingException();
+        }
+
+        if (config.getInternalStorage().getProvider() == ConfigEntity.InternalStorage.Provider.POSTGRES &&
+                Objects.isNull(config.getInternalStorage().getHost())) {
+            throw new ConfigDatabasePropertiesMissingException();
+        }
+
+        switch (config.getInternalStorage().getProvider()) {
+            case SQLITE3 -> {
+                properties.put("quarkus.datasource.jdbc.driver", "org.sqlite.JDBC");
+                properties.put("quarkus.datasource.db-kind", "other");
+                properties.put(
+                        "quarkus.datasource.jdbc.url",
+                        String.format(
+                                "jdbc:sqlite:%s/.%s/internal/database/data.db",
+                                System.getProperty("user.home"),
+                                databaseName.getValue()));
+                properties.put("quarkus.liquibase.change-log", liquibaseSqlite3Config);
+            }
+            case POSTGRES -> {
+                properties.put("quarkus.datasource.db-kind", "postgresql");
+                properties.put(
+                        "quarkus.datasource.jdbc.url",
+                        String.format("jdbc:postgresql://%s/postgres", config.getInternalStorage().getHost()));
+                properties.put("quarkus.liquibase.change-log", liquibasePostgresConfig);
+            }
+        }
+
+        properties.put("quarkus.datasource.username", config.getInternalStorage().getUsername());
+        properties.put("quarkus.datasource.password", config.getInternalStorage().getPassword());
+
+        return Collections.singletonList(
+                new PropertiesConfigSource(
+                        properties,
+                        com.objectstorage.service.integration.properties.security.SecurityPropertiesConfigService.class.getSimpleName(),
+                        290));
+    }
+
+    @Override
+    public OptionalInt getPriority() {
+        return OptionalInt.of(290);
+    }
+}
diff --git a/...bjectstorage/service/integration/properties/security/SecurityPropertiesConfigService.java b/...bjectstorage/service/integration/properties/security/SecurityPropertiesConfigService.java
@@ -24,14 +24,14 @@ public class SecurityPropertiesConfigService implements ConfigSourceFactory {
     @Override
     @SneakyThrows
     public Iterable<ConfigSource> getConfigSources(final ConfigSourceContext context) {
-        final ConfigValue value = context.getValue("config.location");
-        if (value == null || value.getValue() == null) {
+        final ConfigValue configLocation = context.getValue("config.location");
+        if (Objects.isNull(configLocation) || Objects.isNull(configLocation.getValue())) {
             return Collections.emptyList();
         }
 
         Properties properties = new Properties();
 
-        ConfigEntity config = ConfigConfigurationHelper.readConfig(value.getValue(), false);
+        ConfigEntity config = ConfigConfigurationHelper.readConfig(configLocation.getValue(), false);
         if (Objects.isNull(config)) {
             return Collections.emptyList();
         }

diff --git a/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java
@@ -1,7 +1,5 @@
 package com.objectstorage.service.state.watcher;
 
-import org.apache.commons.io.FileUtils;
-
 /**
  * Service used to track state metrics for the current session in the application.
  */
@@ -39,7 +37,10 @@ public void increaseUploadedFilesSize(Integer value) {
      */
     public Double getAverageFileSize() {
         if (filesUploadCounter > 0) {
-            return Double.valueOf(uploadedFilesSize) / Double.valueOf(filesUploadCounter) / 1024 / 1024;
+            return (Double.valueOf(uploadedFilesSize) /
+                    Double.valueOf(filesUploadCounter)) /
+                    (double) 1024 /
+                    (double) 1024;
         }
 
         return (double) 0;

diff --git a/api-server/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceFactory b/api-server/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceFactory
@@ -1 +1,2 @@
-com.objectstorage.service.integration.properties.security.SecurityPropertiesConfigService
+com.objectstorage.service.integration.properties.database.DatabasePropertiesConfigService
+com.objectstorage.service.integration.properties.security.SecurityPropertiesConfigService
-Original file line number
+Diff line change
@@ Expand Up @@
                     } catch (ContentApplicationRetrievalFailureException e) {
                         StateService.getBackupProcessorGuard().unlock();
-                        logger.error(e.getMessage());
                         return;
                     }
@@ Expand Down @@