Add a shared file system to the kernel

docs/articles/Core/FileSystem.md

@@ -0,0 +1,22 @@
+---
+uid: FileSystem
+---
+# FileSystem
+
+````mermaid
+classDiagram
+    MoryxFile <|-- Blob
+    MoryxFile <|-- Tree
+    Tree --> MoryxFile    
+    OwnerFile --> Tree
+    class MoryxFileSystem{
+        -string Directory
+        +WriteBlob()
+        +WriteTree()
+        +ReadBlob()
+        +ReadTree()
+    }
+    class MoryxFile {
+        +String Hash
+    }
+````

src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs

@@ -0,0 +1,67 @@
+using Microsoft.Extensions.Logging;
+using Moryx.Logging;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Moryx.Runtime.Kernel.FileSystem
+{
+    internal class HashPath
+    {
+        public string Hash { get; private set; }
+
+        public string DirectoryName { get; private set; }
+
+        public string FileName { get; private set; }
+
+        private HashPath()
+        {
+        }
+
+        public static HashPath FromStream(Stream stream) =>
+            BuildPath(HashFromStream(stream));
+
+        public static HashPath FromHash(string hash) =>
+            BuildPath(hash);
+
+        public string FilePath(string storagePath) =>
+            Path.Combine(storagePath, DirectoryName, FileName);
+
+        public string DirectoryPath(string storagePath) =>
+            Path.Combine(storagePath, DirectoryName);
+
+        private static HashPath BuildPath(string hash)
+        {
+            return new HashPath
+            {
+                Hash = hash,
+                DirectoryName = hash.Substring(0, 2),
+                FileName = hash.Substring(2)
+            };
+        }
+
+        private static string HashFromStream(Stream stream)
+        {
+            string name;
+            using (var hashing = SHA256.Create())
+            {
+                stream.Position = 0;
+
+                var hash = hashing.ComputeHash(stream);
+                var nameBuilder = new StringBuilder(hash.Length * 2);
+                foreach (var hashByte in hash)
+                {
+                    nameBuilder.AppendFormat("{0:X2}", hashByte);
+                }
+                name = nameBuilder.ToString();
+
+                stream.Position = 0;
+            }
+
+            return name;
+        }
+    }
+}

src/Moryx.Runtime.Kernel/FileSystem/MoryxFileSystem.cs

@@ -0,0 +1,239 @@
+using Castle.MicroKernel.Registration;
+using Microsoft.Extensions.Logging;
+using Moryx.FileSystem;
+using Moryx.Logging;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices.ComTypes;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Moryx.Runtime.Kernel.FileSystem
+{
+    internal class MoryxFileSystem : IMoryxFileSystem
+    {
+        private string _fsDirectory;
+        private string _ownerFilesDirectory;
+        private readonly ILogger _logger;
+
+        public MoryxFileSystem(ILoggerFactory loggerFactory)
+        {
+            _logger = loggerFactory.CreateLogger(nameof(MoryxFileSystem));
+        }
+
+        public void SetBasePath(string basePath = "fs")
+        {
+            _fsDirectory = Path.Combine(Directory.GetCurrentDirectory(), basePath);
+            _ownerFilesDirectory = Path.Combine(_fsDirectory, "owners");
+        }
+
+        public void LoadTrees()
+        {
+            // Load all trees from the owner directory
+            var ownerFiles = Directory.EnumerateFiles(_ownerFilesDirectory);
+
+        }
+
+        public async Task<string> WriteBlobAsync(Stream stream)
+        {
+            var hashPath = HashPath.FromStream(stream);
+
+            // Create directory if necessary
+            var targetPath = hashPath.DirectoryPath(_fsDirectory);
+            try
+            {
+                if (!Directory.Exists(targetPath))
+                    Directory.CreateDirectory(targetPath);
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, _logger, _fsDirectory);
+            }
+
+            var fileName = hashPath.FilePath(_fsDirectory);
+            if (File.Exists(fileName))
+                return hashPath.Hash;
+
+            // Write file
+            try
+            {
+                using var fileStream = new FileStream(fileName, FileMode.Create);
+                await stream.CopyToAsync(fileStream);
+                await fileStream.FlushAsync();
+                stream.Position = 0;
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, _logger, fileName);
+            }
+
+            return hashPath.Hash;
+        }
+        public async Task<string> WriteTreeAsync(IReadOnlyList<MoryxFileMetadata> metadata)
+        {
+            // Convert metadata to lines 
+            var lines = metadata.Select(md => md.ToString()).ToList();
+            var stream = new MemoryStream();
+            using (var sw = new StreamWriter(stream))
+            {
+                foreach (var line in lines)
+                    sw.WriteLine(line);
+                await sw.FlushAsync();
+            }
+
+            return await WriteBlobAsync(stream);
+        }
+
+        public async Task<string> WriteBlobAsync(Stream fileStream, MoryxFileMetadata metadata, string ownerKey)
+        {
+            // Create file first
+            var hash = await WriteBlobAsync(fileStream);
+            metadata.Hash = hash;
+
+            // Read current owner tree
+            var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey);
+            var ownerTree = File.ReadAllText(ownerFile);
+            var tree = ReadExtensibleTree(ownerTree);
+
+            // Add to owner tree/replace hash and write new
+            var exisiting = tree.FirstOrDefault();
+            tree.Add(metadata);
+            var treeHash = await WriteTreeAsync(tree);
+            File.WriteAllText(ownerFile, treeHash);
+
+            return hash;
+        }
+
+        public Stream ReadBlob(string hash)
+        {
+            var path = HashPath.FromHash(hash).FilePath(_fsDirectory);
+            return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null;
+        }
+
+
+        public MoryxFileTree ReadTreeByOwner(string ownerKey)
+        {
+            // read hash from owner file
+            var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey);
+            var ownerTree = File.ReadAllText(ownerFile);
+
+            return ReadExtensibleTree(ownerTree);
+        }
+
+        private MoryxFileTree ReadExtensibleTree(string hash)
+        {
+            // Read tree from hash
+            var stream = ReadBlob(hash);
+            var files = new List<MoryxFile>();
+            using (var sr = new StreamReader(stream))
+            {
+                var line = sr.ReadLine();
+                files.Add(FileFromLine(line));
+            }
+
+            return new MoryxFileTree(files);
+        }
+
+        public bool RemoveFile(string hash, string ownerKey)
+        {
+            if (!IsOwner(hash, ownerKey))
+                return false;
+
+            // Delete file if found
+            var hashPath = HashPath.FromHash(hash);
+            var filePath = hashPath.FilePath(_fsDirectory);
+            if (!File.Exists(filePath))
+                return false;
+            RemoveFile(filePath, _logger);
+
+            // Check if subdirectory is empty and remove
+            var directory = hashPath.DirectoryPath(_fsDirectory);
+            CleanUpDirectory(directory, _logger);
+
+            // TODO: Remove file from owner list
+
+            return true;
+        }
+
+        private bool IsOwner(string hash, string ownerFile)
+        {
+            var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerFile);
+            using (var reader = new StreamReader(ownerFilePath))
+            {
+                string line;
+                while ((line = reader.ReadLine()) != null)
+                {
+                    if (line.Contains(hash))
+                        return true;
+                }
+            }
+            return false;
+        }
+
+        private void RemoveFile(string filePath, ILogger logger)
+        {
+            try
+            {
+                File.Delete(filePath);
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, logger, filePath);
+            }
+        }
+
+        private void CleanUpDirectory(string directoryPath, ILogger logger)
+
+        {
+            try
+            {
+                if (Directory.GetFiles(directoryPath).Length == 0)
+                    Directory.Delete(directoryPath);
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, logger, directoryPath);
+            }
+        }
+
+        private Exception LoggedException(Exception e, ILogger logger, string cause)
+        {
+            switch (e)
+            {
+                case UnauthorizedAccessException unauthorizedAccessException:
+                    logger.LogError("Error: {0}. You do not have the required permission to manipulate the file {1}.", e.Message, cause); // ToDo
+                    return unauthorizedAccessException;
+                case ArgumentException argumentException:
+                    logger.LogError("Error: {0}. The path {1} contains invalid characters such as \", <, >, or |.", e.Message, cause);
+                    return argumentException;
+                case IOException iOException:
+                    logger.LogError("Error: {0}. An I/O error occurred while opening the file {1}.", e.Message, cause);
+                    return iOException;
+                default:
+                    logger.LogError("Unspecified error on file system access: {0}", e.Message);
+                    return e;
+            }
+        }
+
+        private static string FileToLine(MoryxFile file)
+        {
+            return $"{(int)file.Mode} {file.FileType.ToString().ToLower()} {file.MimeType} {file.Hash} {file.FileName}";
+        }
+
+        private static MoryxFile FileFromLine(string line)
+        {
+            var parts = line.Split(' ');
+
+            var file = parts[1] == FileType.Blob.ToString().ToLower()
+                ? new MoryxFile() : new MoryxFileTree();
+            file.Mode = (MoryxFileMode)int.Parse(parts[0]);
+            file.MimeType = parts[2];
+            file.Hash = parts[3];
+            file.FileName = string.Join(" ", parts.Skip(4));
+
+            return file;
+        }
+    }
+}

src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs

@@ -4,6 +4,8 @@
 using Microsoft.Extensions.DependencyInjection;
 using Moryx.Configuration;
 using Moryx.Container;
+using Moryx.FileSystem;
+using Moryx.Runtime.Kernel.FileSystem;
 using Moryx.Runtime.Modules;
 using Moryx.Threading;
 using System;
@@ -30,6 +32,10 @@ public static void AddMoryxKernel(this IServiceCollection serviceCollection)
             serviceCollection.AddSingleton<ModuleManager>();
             serviceCollection.AddSingleton<IModuleManager>(x => x.GetRequiredService<ModuleManager>());
 
+            // Register module manager
+            serviceCollection.AddSingleton<MoryxFileSystem>();
+            serviceCollection.AddSingleton<IMoryxFileSystem>(x => x.GetRequiredService<MoryxFileSystem>());
+
             // Register parallel operations
             serviceCollection.AddTransient<IParallelOperations, ParallelOperations>();
 
@@ -87,6 +93,19 @@ public static IConfigManager UseMoryxConfigurations(this IServiceProvider servic
             return configManager;
         }
 
+        /// <summary>
+        /// Use moryx file system and configure base directory
+        /// </summary>
+        /// <returns></returns>
+        public static IMoryxFileSystem UseMoryxFileSystem(this IServiceProvider serviceProvider, string path)
+        {
+            var fileSystem = serviceProvider.GetRequiredService<MoryxFileSystem>();
+            if (!Directory.Exists(path))
+                Directory.CreateDirectory(path);
+            fileSystem.SetBasePath(path);
+            return fileSystem;
+        }
+
         private static IModuleManager _moduleManager;
         /// <summary>
         /// Boot system and start all modules

src/Moryx/FileSystem/FileType.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Moryx.FileSystem
+{
+    /// <summary>
+    /// Moryx file types in owner tree file
+    /// </summary>
+    public enum FileType
+    {
+        Blob = 0,
+
+        Tree = 1,
+    }
+}