GH-391 Add Support for Other Storage Solutions

.run/Reposilite.run.xml

@@ -11,7 +11,7 @@
     </method>
   </configuration>
   <configuration default="false" name="Reposilite" type="Application" factoryName="Application">
-    <option name="ALTERNATIVE_JRE_PATH" value="C:\Program Files\Java\jdk-16" />
+    <option name="ALTERNATIVE_JRE_PATH" value="8" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
     <option name="MAIN_CLASS_NAME" value="org.panda_lang.reposilite.ReposiliteLauncher" />
     <module name="reposilite" />

reposilite-backend/pom.xml

@@ -40,6 +40,18 @@
         <jetty.version>9.4.40.v20210413</jetty.version>
     </properties>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bom</artifactId>
+                <version>2.15.15</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
         <!-- OpenAPI -->
         <dependency>
@@ -247,6 +259,12 @@
             <version>5.7.1</version>
             <scope>test</scope>
         </dependency>
+
+        <!-- S3 -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+        </dependency>
     </dependencies>
 
     <repositories>

reposilite-backend/src/main/java/org/panda_lang/reposilite/Reposilite.java

@@ -28,28 +28,21 @@
 import org.panda_lang.reposilite.resource.FrontendProvider;
 import org.panda_lang.reposilite.metadata.MetadataConfiguration;
 import org.panda_lang.reposilite.metadata.MetadataService;
-import org.panda_lang.reposilite.repository.DeployService;
-import org.panda_lang.reposilite.repository.LookupService;
-import org.panda_lang.reposilite.repository.ProxyService;
-import org.panda_lang.reposilite.repository.RepositoryAuthenticator;
-import org.panda_lang.reposilite.repository.RepositoryService;
+import org.panda_lang.reposilite.repository.*;
 import org.panda_lang.reposilite.stats.StatsConfiguration;
 import org.panda_lang.reposilite.stats.StatsService;
+import org.panda_lang.reposilite.storage.FileSystemStorageProvider;
+import org.panda_lang.reposilite.storage.StorageProvider;
 import org.panda_lang.reposilite.utils.RunUtils;
 import org.panda_lang.reposilite.utils.TimeUtils;
 import org.panda_lang.utilities.commons.ValidationUtils;
 import org.panda_lang.utilities.commons.function.ThrowingRunnable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public final class Reposilite {
@@ -58,11 +51,10 @@ public final class Reposilite {
 
     private final boolean servlet;
     private final AtomicBoolean alive;
-    private final ExecutorService ioService;
-    private final ScheduledExecutorService retryService;
-    private final File configurationFile;
-    private final File workingDirectory;
+    private final Path configurationFile;
+    private final Path workingDirectory;
     private final boolean testEnvEnabled;
+    private final FileSystemStorageProvider storageProvider;
     private final Configuration configuration;
     private final ReposiliteContextFactory contextFactory;
     private final ReposiliteExecutor executor;
@@ -83,33 +75,32 @@ public final class Reposilite {
     private final Thread shutdownHook;
     private long uptime;
 
-    Reposilite(String configurationFile, String workingDirectory, boolean servlet, boolean testEnv) {
+    Reposilite(Path configurationFile, Path workingDirectory, boolean servlet, boolean testEnv) {
         ValidationUtils.notNull(configurationFile, "Configuration file cannot be null. To use default configuration file, provide empty string");
         ValidationUtils.notNull(workingDirectory, "Working directory cannot be null. To use default working directory, provide empty string");
 
         this.servlet = servlet;
         this.alive = new AtomicBoolean(false);
-        this.ioService = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 1, TimeUnit.SECONDS, new SynchronousQueue<>());
-        this.retryService = Executors.newSingleThreadScheduledExecutor();
-        this.configurationFile = new File(configurationFile);
-        this.workingDirectory = new File(workingDirectory);
+        this.configurationFile = configurationFile;
+        this.workingDirectory = workingDirectory;
         this.testEnvEnabled = testEnv;
 
-        this.configuration = ConfigurationLoader.tryLoad(configurationFile, workingDirectory);
+        this.configuration = ConfigurationLoader.tryLoad(configurationFile);
+        this.storageProvider = FileSystemStorageProvider.of(Paths.get(""), this.configuration.diskQuota);
         this.contextFactory = new ReposiliteContextFactory(configuration.forwardedIp);
         this.failureService = new FailureService();
         this.executor = new ReposiliteExecutor(testEnvEnabled, failureService);
-        this.tokenService = new TokenService(workingDirectory);
-        this.statsService = new StatsService(workingDirectory, failureService, ioService, retryService);
-        this.repositoryService = new RepositoryService(workingDirectory, configuration.diskQuota, ioService, retryService, failureService);
+        this.tokenService = new TokenService(workingDirectory, storageProvider);
+        this.statsService = new StatsService(workingDirectory, failureService, storageProvider);
+        this.repositoryService = new RepositoryService();
         this.metadataService = new MetadataService(failureService);
 
         this.authenticator = new Authenticator(repositoryService, tokenService);
         this.repositoryAuthenticator = new RepositoryAuthenticator(configuration.rewritePathsEnabled, authenticator, repositoryService);
         this.authService = new AuthService(authenticator);
         this.deployService = new DeployService(configuration.deployEnabled, configuration.rewritePathsEnabled, authenticator, repositoryService, metadataService);
-        this.lookupService = new LookupService(authenticator, repositoryAuthenticator, metadataService, repositoryService, ioService, failureService);
-        this.proxyService = new ProxyService(configuration.storeProxied, configuration.proxyPrivate, configuration.proxyConnectTimeout, configuration.proxyReadTimeout, configuration.rewritePathsEnabled, configuration.proxied, ioService, failureService, repositoryService);
+        this.lookupService = new LookupService(repositoryAuthenticator, repositoryService);
+        this.proxyService = new ProxyService(configuration.storeProxied, configuration.proxyPrivate, configuration.proxyConnectTimeout, configuration.proxyReadTimeout, configuration.proxied, repositoryService, failureService, storageProvider);
         this.frontend = FrontendProvider.load(configuration);
         this.reactiveHttpServer = new ReposiliteHttpServer(this, servlet);
         this.console = new Console(System.in, failureService);
@@ -138,8 +129,8 @@ public void load() throws Exception {
         }
 
         getLogger().info("Platform: " + System.getProperty("java.version") + " (" + System.getProperty("os.name") + ")");
-        getLogger().info("Configuration: " + configurationFile.getAbsolutePath());
-        getLogger().info("Working directory: " + workingDirectory.getAbsolutePath());
+        getLogger().info("Configuration: " + configurationFile.toAbsolutePath());
+        getLogger().info("Working directory: " + workingDirectory.toAbsolutePath());
         getLogger().info("");
 
         this.alive.set(true);
@@ -202,8 +193,7 @@ public synchronized void shutdown() throws Exception {
 
         reactiveHttpServer.stop();
         statsService.saveStats();
-        ioService.shutdownNow();
-        retryService.shutdownNow();
+        storageProvider.shutdown();
         console.stop();
         executor.stop();
     }
@@ -288,20 +278,15 @@ public ReposiliteExecutor getExecutor() {
         return executor;
     }
 
-    public ScheduledExecutorService getRetryService() {
-        return retryService;
-    }
-
-    public ExecutorService getIoService() {
-        return ioService;
-    }
-
-    public File getWorkingDirectory() {
+    public Path getWorkingDirectory() {
         return workingDirectory;
     }
 
     public static Logger getLogger() {
         return LOGGER;
     }
 
+    public StorageProvider getStorageProvider() {
+        return storageProvider;
+    }
 }

reposilite-backend/src/main/java/org/panda_lang/reposilite/ReposiliteHttpServer.java

@@ -62,7 +62,6 @@ void start(Configuration configuration, Runnable onStart) {
                 reposilite.getFailureService());
 
         LookupApiEndpoint lookupApiEndpoint = new LookupApiEndpoint(
-                configuration.rewritePathsEnabled,
                 reposilite.getContextFactory(),
                 reposilite.getRepositoryAuthenticator(),
                 reposilite.getRepositoryService());
@@ -105,14 +104,15 @@ private Javalin create(Configuration configuration) {
                 : Javalin.create(config -> configure(configuration, config));
     }
 
+    @SuppressWarnings("deprecation")
     private void configure(Configuration configuration, JavalinConfig config) {
         Server server = new Server();
 
         if (configuration.sslEnabled) {
             Reposilite.getLogger().info("Enabling SSL connector at ::" + configuration.sslPort);
 
             SslContextFactory sslContextFactory = new SslContextFactory.Server();
-            sslContextFactory.setKeyStorePath(configuration.keyStorePath.replace("${WORKING_DIRECTORY}", reposilite.getWorkingDirectory().getAbsolutePath()));
+            sslContextFactory.setKeyStorePath(configuration.keyStorePath.replace("${WORKING_DIRECTORY}", reposilite.getWorkingDirectory().toAbsolutePath().toString()));
             sslContextFactory.setKeyStorePassword(configuration.keyStorePassword);
 
             ServerConnector sslConnector = new ServerConnector(server, sslContextFactory);

reposilite-backend/src/main/java/org/panda_lang/reposilite/ReposiliteLauncher.java

@@ -17,13 +17,13 @@
 package org.panda_lang.reposilite;
 
 import org.panda_lang.reposilite.utils.RunUtils;
-import org.panda_lang.utilities.commons.StringUtils;
 import org.panda_lang.utilities.commons.console.Effect;
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
-import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Optional;
 
 @Command(name = "reposilite", version = "Reposilite " + ReposiliteConstants.VERSION)
@@ -57,26 +57,29 @@ public static Optional<Reposilite> create(String... args) {
         }
 
         if (launcher.versionInfoRequested) {
-            System.out.println("Reposilite " + ReposiliteConstants.VERSION);;
+            System.out.println("Reposilite " + ReposiliteConstants.VERSION);
             return Optional.empty();
         }
 
-        return Optional.of(create(launcher.configurationFile, launcher.workingDirectory, false, launcher.testEnv));
+        return Optional.of(create(launcher.workingDirectory, launcher.configurationFile, false, launcher.testEnv));
     }
 
-    public static Reposilite create(String configurationFile, String workingDirectory, boolean servlet, boolean testEnv) {
+    public static Reposilite create(String workingDirectoryString, String configurationFileName, boolean servlet, boolean testEnv) {
         Reposilite.getLogger().info("");
         Reposilite.getLogger().info(Effect.GREEN + "Reposilite " + Effect.RESET + ReposiliteConstants.VERSION);
         Reposilite.getLogger().info("");
 
-        if (StringUtils.isEmpty(workingDirectory)) {
-            workingDirectory = ".";
-        }
+        Path workingDirectory = Paths.get("");
 
-        if (StringUtils.isEmpty(configurationFile)) {
-            configurationFile = new File(workingDirectory, ReposiliteConstants.CONFIGURATION_FILE_NAME).getAbsolutePath();
+        if (workingDirectoryString != null && !workingDirectoryString.isEmpty()) {
+            workingDirectory = Paths.get(workingDirectoryString);
         }
 
+        Path configurationFile = workingDirectory.resolve(configurationFileName == null || configurationFileName.isEmpty()
+                ? ReposiliteConstants.CONFIGURATION_FILE_NAME
+                : configurationFileName
+        );
+
         return new Reposilite(configurationFile, workingDirectory, servlet, testEnv);
     }
 

reposilite-backend/src/main/java/org/panda_lang/reposilite/ReposiliteUtils.java

@@ -31,37 +31,43 @@ private ReposiliteUtils() { }
      * <ul>
      *     <li>Remove root slash</li>
      *     <li>Remove illegal path modifiers like .. and ~</li>
-     *     <li>Insert repository name if missing</li>
      * </ul>
      *
-     * @param rewritePathsEnabled determines if path rewriting is enabled
      * @param uri the uri to process
      * @return the normalized uri
      */
-    public static Option<String> normalizeUri(boolean rewritePathsEnabled, RepositoryService repositoryService, String uri) {
+    public static Option<String> normalizeUri(String uri) {
+        if (uri.contains("..") || uri.contains("~") || uri.contains(":") || uri.contains("\\")) {
+            return Option.none();
+        }
+
         if (uri.startsWith("/")) {
             uri = uri.substring(1);
         }
 
-        if (uri.contains("..") || uri.contains("~") || uri.contains(":") || uri.contains("\\")) {
-            return Option.none();
-        }
+        return Option.of(uri);
+    }
+
+    public static Option<Repository> getRepository(boolean rewritePathsEnabled, RepositoryService repositoryService, String uri) {
+        String repositoryName = uri;
 
-        if (!rewritePathsEnabled) {
-            return Option.of(uri);
+        if (repositoryName.startsWith("/")) {
+            repositoryName = repositoryName.substring(1);
         }
 
-        if (StringUtils.countOccurrences(uri, "/") <= 1) {
-            return Option.of(uri);
+        if (repositoryName.contains("..") || repositoryName.contains("~") || repositoryName.contains(":") || repositoryName.contains("\\")) {
+            return Option.none();
         }
 
-        for (Repository repository : repositoryService.getRepositories()) {
-            if (uri.startsWith(repository.getName())) {
-                return Option.of(uri);
-            }
+        String repository = StringUtils.countOccurrences(repositoryName, "/") > 0
+                ? repositoryName.substring(0, repositoryName.indexOf('/'))
+                : repositoryName;
+
+        if (rewritePathsEnabled && repositoryService.getRepository(repository) == null) {
+            repository = repositoryService.getPrimaryRepository().getName();
         }
 
-        return Option.of(repositoryService.getPrimaryRepository().getName() + "/" + uri);
+        return Option.of(repositoryService.getRepository(repository));
     }
 
 }

reposilite-backend/src/main/java/org/panda_lang/reposilite/auth/Session.java

@@ -19,6 +19,7 @@
 import org.panda_lang.reposilite.repository.Repository;
 import org.panda_lang.utilities.commons.StringUtils;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -27,9 +28,9 @@ public final class Session {
     public static final String WILDCARD = "*";
 
     private final Token token;
-    private final List<Repository> repositories;
+    private final Collection<Repository> repositories;
 
-    public Session(Token token, List<Repository> repositories) {
+    public Session(Token token, Collection<Repository> repositories) {
         this.token = token;
         this.repositories = repositories;
     }
@@ -59,7 +60,7 @@ public boolean hasPermissionTo(String path) {
         return path.startsWith(tokenPath) || path.startsWith(tokenPath + "/");
     }
 
-    public List<Repository> getRepositories() {
+    public Collection<Repository> getRepositories() {
         return repositories;
     }
 

reposilite-backend/src/main/java/org/panda_lang/reposilite/auth/TokenCollection.java

@@ -17,11 +17,12 @@
 package org.panda_lang.reposilite.auth;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
 
 public final class TokenCollection implements Serializable {
 
-    List<Token> tokens;
+    List<Token> tokens = new ArrayList<>();
 
     public void setTokens(List<Token> tokens) {
         this.tokens = tokens;

reposilite-backend/src/main/java/org/panda_lang/reposilite/auth/TokenService.java

@@ -16,12 +16,14 @@
 
 package org.panda_lang.reposilite.auth;
 
+import org.panda_lang.reposilite.storage.StorageProvider;
 import org.panda_lang.utilities.commons.collection.Pair;
 import org.panda_lang.utilities.commons.function.Option;
 import org.panda_lang.utilities.commons.function.Result;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.security.SecureRandom;
 import java.util.Base64;
 import java.util.Collection;
@@ -37,8 +39,8 @@ public final class TokenService {
     private final Map<String, Token> tokens = new HashMap<>();
     private final TokenStorage database;
 
-    public TokenService(String workingDirectory) {
-        this.database = new TokenStorage(this, workingDirectory);
+    public TokenService(Path workingDirectory, StorageProvider storageProvider) {
+        this.database = new TokenStorage(this, workingDirectory, storageProvider);
     }
 
     public void loadTokens() throws IOException {