diff --git a/docs/articles/Core/FileSystem.md b/docs/articles/Core/FileSystem.md new file mode 100644 index 000000000..a382dfe27 --- /dev/null +++ b/docs/articles/Core/FileSystem.md @@ -0,0 +1,22 @@ +--- +uid: FileSystem +--- +# FileSystem + +````mermaid +classDiagram + MoryxFile <|-- Blob + MoryxFile <|-- Tree + Tree --> MoryxFile + OwnerFile --> Tree + class MoryxFileSystem{ + -string Directory + +WriteBlob() + +WriteTree() + +ReadBlob() + +ReadTree() + } + class MoryxFile { + +String Hash + } +```` \ No newline at end of file diff --git a/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs b/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs new file mode 100644 index 000000000..27ad90967 --- /dev/null +++ b/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.Logging; +using Moryx.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.Runtime.Kernel.FileSystem +{ + internal class HashPath + { + public string Hash { get; private set; } + + public string DirectoryName { get; private set; } + + public string FileName { get; private set; } + + private HashPath() + { + } + + public static HashPath FromStream(Stream stream) => + BuildPath(HashFromStream(stream)); + + public static HashPath FromHash(string hash) => + BuildPath(hash); + + public string FilePath(string storagePath) => + Path.Combine(storagePath, DirectoryName, FileName); + + public string DirectoryPath(string storagePath) => + Path.Combine(storagePath, DirectoryName); + + private static HashPath BuildPath(string hash) + { + return new HashPath + { + Hash = hash, + DirectoryName = hash.Substring(0, 2), + FileName = hash.Substring(2) + }; + } + + private static string HashFromStream(Stream stream) + { + string name; + using (var hashing = SHA256.Create()) + { + stream.Position = 0; + + var hash = hashing.ComputeHash(stream); + var nameBuilder = new StringBuilder(hash.Length * 2); + foreach (var hashByte in hash) + { + nameBuilder.AppendFormat("{0:X2}", hashByte); + } + name = nameBuilder.ToString(); + + stream.Position = 0; + } + + return name; + } + } +} diff --git a/src/Moryx.Runtime.Kernel/FileSystem/MoryxFileSystem.cs b/src/Moryx.Runtime.Kernel/FileSystem/MoryxFileSystem.cs new file mode 100644 index 000000000..18a40c0e7 --- /dev/null +++ b/src/Moryx.Runtime.Kernel/FileSystem/MoryxFileSystem.cs @@ -0,0 +1,218 @@ +using Microsoft.Extensions.Logging; +using Moryx.FileSystem; +using Moryx.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.Runtime.Kernel.FileSystem +{ + internal class MoryxFileSystem : IMoryxFileSystem + { + private string _fsDirectory; + private string _ownerFilesDirectory; + private readonly ILogger _logger; + + public MoryxFileSystem(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(nameof(MoryxFileSystem)); + } + + public void SetBasePath(string basePath = "fs") + { + _fsDirectory = Path.Combine(Directory.GetCurrentDirectory(), basePath); + _ownerFilesDirectory = Path.Combine(_fsDirectory, "owners"); + } + + public async Task WriteBlob(Stream stream) + { + var hashPath = HashPath.FromStream(stream); + + // Create directory if necessary + var targetPath = hashPath.DirectoryPath(_fsDirectory); + try + { + if (!Directory.Exists(targetPath)) + Directory.CreateDirectory(targetPath); + } + catch (Exception e) + { + throw LoggedException(e, _logger, _fsDirectory); + } + + var fileName = hashPath.FilePath(_fsDirectory); + if (File.Exists(fileName)) + return hashPath.Hash; + + // Write file + try + { + using var fileStream = new FileStream(fileName, FileMode.Create); + await stream.CopyToAsync(fileStream); + fileStream.Flush(); + stream.Position = 0; + } + catch (Exception e) + { + throw LoggedException(e, _logger, fileName); + } + + return hashPath.Hash; + } + public Task WriteTree(IReadOnlyList metadata) + { + // Convert metadata to lines + var lines = metadata.Select(md => md.ToString()).ToList(); + var stream = new MemoryStream(); + using (var sw = new StreamWriter(stream)) + { + foreach (var line in lines) + sw.WriteLine(line); + sw.Flush(); + } + + return WriteBlob(stream); + } + + public async Task WriteBlob(Stream fileStream, string ownerKey, MoryxFileMetadata metadata) + { + // Create file first + var hash = await WriteBlob(fileStream); + metadata.Hash = hash; + + // Read current owner tree + var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey); + var ownerTree = File.ReadAllText(ownerFile); + var tree = ReadExtensibleTree(ownerTree); + + // Add to owner tree and write new + tree.Add(metadata); + var treeHash = WriteTree(tree); + + // Update owner file + var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerKey); + if (File.Exists(ownerFilePath)) + await File.WriteAllLinesAsync(ownerFilePath, new[] { hash }); + else + await File.AppendAllLinesAsync(ownerFilePath, new[] { hash }); + + return hash; + } + + public Stream ReadBlob(string hash) + { + var path = HashPath.FromHash(hash).FilePath(_fsDirectory); + return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null; + } + + public IReadOnlyList ReadTree(string hash) => ReadExtensibleTree(hash); + + + public IReadOnlyList ReadTreeByOwner(string ownerKey) + { + // read hash from owner file + var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey); + var ownerTree = File.ReadAllText(ownerFile); + + return ReadExtensibleTree(ownerTree); + } + + private List ReadExtensibleTree(string hash) + { + // Read tree from hash + var stream = ReadBlob(hash); + var metadata = new List(); + using (var sr = new StreamReader(stream)) + { + var line = sr.ReadLine(); + metadata.Add(MoryxFileMetadata.FromLine(line)); + } + + return metadata; + } + + public bool RemoveFile(string hash, string ownerKey) + { + if (!IsOwner(hash, ownerKey)) + return false; + + // Delete file if found + var hashPath = HashPath.FromHash(hash); + var filePath = hashPath.FilePath(_fsDirectory); + if (!File.Exists(filePath)) + return false; + RemoveFile(filePath, _logger); + + // Check if subdirectory is empty and remove + var directory = hashPath.DirectoryPath(_fsDirectory); + CleanUpDirectory(directory, _logger); + + // TODO: Remove file from owner list + + return true; + } + + private bool IsOwner(string hash, string ownerFile) + { + var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerFile); + using (var reader = new StreamReader(ownerFilePath)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + if (line.Contains(searchLine)) + return true; + } + } + return false; + } + + private void RemoveFile(string filePath, ILogger logger) + { + try + { + File.Delete(filePath); + } + catch (Exception e) + { + throw LoggedException(e, logger, filePath); + } + } + + private void CleanUpDirectory(string directoryPath, ILogger logger) + + { + try + { + if (Directory.GetFiles(directoryPath).Length == 0) + Directory.Delete(directoryPath); + } + catch (Exception e) + { + throw LoggedException(e, logger, directoryPath); + } + } + + private Exception LoggedException(Exception e, ILogger logger, string cause) + { + switch (e) + { + case UnauthorizedAccessException unauthorizedAccessException: + logger.LogError("Error: {0}. You do not have the required permission to manipulate the file {1}.", e.Message, cause); // ToDo + return unauthorizedAccessException; + case ArgumentException argumentException: + logger.LogError("Error: {0}. The path {1} contains invalid characters such as \", <, >, or |.", e.Message, cause); + return argumentException; + case IOException iOException: + logger.LogError("Error: {0}. An I/O error occurred while opening the file {1}.", e.Message, cause); + return iOException; + default: + logger.LogError("Unspecified error on file system access: {0}", e.Message); + return e; + } + } + } +} diff --git a/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs b/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs index 83bc5b833..b20a54651 100644 --- a/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs +++ b/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs @@ -4,6 +4,8 @@ using Microsoft.Extensions.DependencyInjection; using Moryx.Configuration; using Moryx.Container; +using Moryx.FileSystem; +using Moryx.Runtime.Kernel.FileSystem; using Moryx.Runtime.Modules; using Moryx.Threading; using System; @@ -30,6 +32,10 @@ public static void AddMoryxKernel(this IServiceCollection serviceCollection) serviceCollection.AddSingleton(); serviceCollection.AddSingleton(x => x.GetRequiredService()); + // Register module manager + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(x => x.GetRequiredService()); + // Register parallel operations serviceCollection.AddTransient(); @@ -87,6 +93,19 @@ public static IConfigManager UseMoryxConfigurations(this IServiceProvider servic return configManager; } + /// + /// Use moryx file system and configure base directory + /// + /// + public static IMoryxFileSystem UseMoryxFileSystem(this IServiceProvider serviceProvider, string path) + { + var fileSystem = serviceProvider.GetRequiredService(); + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); + fileSystem.SetBasePath(path); + return fileSystem; + } + private static IModuleManager _moduleManager; /// /// Boot system and start all modules diff --git a/src/Moryx/FileSystem/IMoryxFileSystem.cs b/src/Moryx/FileSystem/IMoryxFileSystem.cs new file mode 100644 index 000000000..376e0d7c0 --- /dev/null +++ b/src/Moryx/FileSystem/IMoryxFileSystem.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.FileSystem +{ + /// + /// Common file system interface across MORYX modules and components. + /// + public interface IMoryxFileSystem + { + /// + /// Write a file to the file system and receive the hash to access it later + /// + Task WriteBlob(Stream fileStream); + + /// + /// Write a file to the file system and receive the hash to access it later + /// + Task WriteBlob(Stream fileStream, string ownerKey, MoryxFileMetadata metadata); + + /// + /// Write a file to the file system and receive the hash to access it later + /// + Task WriteTree(IReadOnlyList metadata); + + /// + /// Read the file by passing the file system hash + /// + /// + /// + Stream ReadBlob(string hash); + + /// + /// Read a tree file and return the listed files + /// + IReadOnlyList ReadTree(string hash); + + /// + /// Return all files stored under + /// + /// + /// + IReadOnlyList ReadTreeByOwner(string ownerKey); + + /// + /// Remove a file by hash and provided owner key. + /// Files without owner key can not be removed + /// + bool RemoveFile(string hash, string ownerKey); + } +} diff --git a/src/Moryx/FileSystem/MoryxFileMetadata.cs b/src/Moryx/FileSystem/MoryxFileMetadata.cs new file mode 100644 index 000000000..8cba09719 --- /dev/null +++ b/src/Moryx/FileSystem/MoryxFileMetadata.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Moryx.FileSystem +{ + public class MoryxFileMetadata + { + public int Mode { get; set; } + + public FileType FileType { get; set; } + + public string MimeType { get; set; } + + public string Hash { get; set; } + + public string FileName { get; set; } + + public string ToLine() + { + return $"{Mode} {FileType.ToString().ToLower()} {MimeType} {Hash} {FileName}"; + } + + public static MoryxFileMetadata FromLine(string line) + { + var parts = line.Split(' '); + return new MoryxFileMetadata + { + Mode = int.Parse(parts[0]), + FileType = (FileType)Enum.Parse(typeof(FileType), parts[1]), + MimeType = parts[2], + Hash = parts[3], + FileName = string.Join(" ", parts.Skip(4)) + }; + } + } + + /// + /// Moryx file types in owner tree file + /// + public enum FileType + { + Blob = 0, + + Tree = 1, + } +} diff --git a/src/Moryx/Moryx.csproj b/src/Moryx/Moryx.csproj index 9a2ea99dc..5585e660f 100644 --- a/src/Moryx/Moryx.csproj +++ b/src/Moryx/Moryx.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/StartProject.Asp/Program.cs b/src/StartProject.Asp/Program.cs index bc308a172..53db4b406 100644 --- a/src/StartProject.Asp/Program.cs +++ b/src/StartProject.Asp/Program.cs @@ -27,6 +27,7 @@ public static void Main(string[] args) webBuilder.UseStartup(); }).Build(); + host.Services.UseMoryxFileSystem("fs"); host.Services.UseMoryxConfigurations("Config"); var moduleManager = host.Services.GetRequiredService();