From ac851112cedf6073de2861e36a81121e032fe665 Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 22 Feb 2024 01:53:25 -0500 Subject: [PATCH 01/21] Update SamplePlugin.csproj --- SamplePlugin/SamplePlugin.csproj | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 759fe5a96..c4ca5e577 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -8,19 +8,12 @@ true true false - - - ..\Obsidian.ConsoleApp\bin\Debug\$(TargetFramework)\plugins\ - none - false - Off + SamplePlugin + 1.0.0.0 + 1.0.0 - - - - @@ -29,6 +22,7 @@ runtime + @@ -38,4 +32,8 @@ + + + + From d4774803ff23ebc9f3db4351f0e42741c248cba4 Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 22 Feb 2024 19:46:39 -0500 Subject: [PATCH 02/21] Update SamplePlugin.csproj --- SamplePlugin/SamplePlugin.csproj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index c4ca5e577..0e6520526 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -11,7 +11,7 @@ SamplePlugin 1.0.0.0 - 1.0.0 + 1.0.0 @@ -22,7 +22,7 @@ runtime - + @@ -32,8 +32,7 @@ - - + + - From e03a907497b69803e95da8679d7142e7e39ef1a6 Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 29 Feb 2024 19:44:21 -0500 Subject: [PATCH 03/21] PluginProvider refactor --- Obsidian/Plugins/PluginContainer.cs | 46 ++-- Obsidian/Plugins/PluginFileEntry.cs | 45 ++++ Obsidian/Plugins/PluginManager.cs | 24 +- .../PluginProviders/CompiledPluginProvider.cs | 48 ---- .../PluginProviders/IPluginProvider.cs | 8 - .../PluginProviders/PackedPluginProvider.cs | 151 +++++++++++ .../PluginProviders/PluginLoadContext.cs | 33 ++- .../PluginProviders/PluginProviderSelector.cs | 34 --- .../PluginProviders/RemotePluginProvider.cs | 238 ------------------ .../UncompiledPluginProvider.cs | 89 ------- Obsidian/Server.cs | 2 +- SamplePlugin/SamplePlugin.csproj | 18 +- 12 files changed, 266 insertions(+), 470 deletions(-) create mode 100644 Obsidian/Plugins/PluginFileEntry.cs delete mode 100644 Obsidian/Plugins/PluginProviders/CompiledPluginProvider.cs delete mode 100644 Obsidian/Plugins/PluginProviders/IPluginProvider.cs create mode 100644 Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs delete mode 100644 Obsidian/Plugins/PluginProviders/PluginProviderSelector.cs delete mode 100644 Obsidian/Plugins/PluginProviders/RemotePluginProvider.cs delete mode 100644 Obsidian/Plugins/PluginProviders/UncompiledPluginProvider.cs diff --git a/Obsidian/Plugins/PluginContainer.cs b/Obsidian/Plugins/PluginContainer.cs index 9dfbba14b..469fa27b2 100644 --- a/Obsidian/Plugins/PluginContainer.cs +++ b/Obsidian/Plugins/PluginContainer.cs @@ -1,7 +1,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; -using System.Diagnostics.CodeAnalysis; +using System.Collections.Immutable; +using System.IO; using System.Reflection; using System.Runtime.Loader; @@ -13,39 +14,52 @@ public sealed class PluginContainer : IDisposable public IServiceScope ServiceScope { get; internal set; } = default!; - [AllowNull] - public PluginBase Plugin { get; private set; } + public PluginBase? Plugin { get; private set; } public PluginInfo Info { get; } - [AllowNull] - public AssemblyLoadContext LoadContext { get; private set; } + public AssemblyLoadContext? LoadContext { get; private set; } public Assembly PluginAssembly { get; } = default!; public string Source { get; internal set; } = default!; - public string ClassName { get; } = default!; - public bool HasDependencies { get; private set; } = true; public bool IsReady => HasDependencies; public bool Loaded { get; internal set; } - public PluginContainer(PluginInfo info, string source) - { - Info = info; - Source = source; - } + public ImmutableArray FileEntries { get; } - public PluginContainer(PluginBase plugin, PluginInfo info, Assembly assembly, AssemblyLoadContext loadContext, string source) + public PluginContainer(PluginBase plugin, PluginInfo info, Assembly assembly, AssemblyLoadContext loadContext, + string source, IEnumerable fileEntries) { - Plugin = plugin; Info = info; LoadContext = loadContext; Source = source; PluginAssembly = assembly; - pluginType = plugin.GetType(); - ClassName = pluginType.Name; Plugin.Info = Info; + this.FileEntries = fileEntries.ToImmutableArray(); + } + + /// + /// Searches for the specified file. + /// + /// The name of the file you're searching for. + /// Null if the file is not found or the byte array of the file. + public async Task GetFileDataAsync(string fileName) + { + var fileEntry = this.FileEntries.FirstOrDefault(x => Path.GetFileName(x.FullName) == fileName); + if (fileEntry is null) + return null; + + await using var fs = new FileStream(this.Source, FileMode.Open); + + fs.Seek(fileEntry.Offset, SeekOrigin.Begin); + + var data = new byte[fileEntry.CompressedLength]; + + await fs.ReadAsync(data); + + return data; } /// diff --git a/Obsidian/Plugins/PluginFileEntry.cs b/Obsidian/Plugins/PluginFileEntry.cs new file mode 100644 index 000000000..53fd2863a --- /dev/null +++ b/Obsidian/Plugins/PluginFileEntry.cs @@ -0,0 +1,45 @@ +using Org.BouncyCastle.Crypto; +using System.Buffers; +using System.IO; +using System.IO.Compression; + +namespace Obsidian.Plugins; +public sealed class PluginFileEntry +{ + public required string FullName { get; init; } + + public required int Length { get; init; } + + public required int CompressedLength { get; init; } + + public required int Offset { get; set; } + + public bool IsCompressed => CompressedLength < Length; + + public async Task GetDataAsync(FileStream packedPluginFile) + { + packedPluginFile.Seek(this.Offset, SeekOrigin.Begin); + + if (!this.IsCompressed) + { + var mem = new byte[this.Length]; + + return await packedPluginFile.ReadAsync(mem) != this.Length ? throw new DataLengthException() : mem; + } + + var compressedData = ArrayPool.Shared.Rent(this.CompressedLength); + + if (await packedPluginFile.ReadAsync(compressedData.AsMemory(0, this.CompressedLength)) != this.CompressedLength) + throw new DataLengthException(); + + await using var ms = new MemoryStream(compressedData); + await using var ds = new DeflateStream(ms, CompressionMode.Decompress); + await using var outStream = new MemoryStream(); + + await ds.CopyToAsync(outStream); + + ArrayPool.Shared.Return(compressedData); + + return outStream.ToArray(); + } +} diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 4b9b0cafa..e36767ac9 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -8,6 +8,7 @@ using Obsidian.Plugins.ServiceProviders; using Obsidian.Registries; using Obsidian.Services; +using System.Collections.Immutable; using System.Reflection; namespace Obsidian.Plugins; @@ -18,13 +19,19 @@ public sealed class PluginManager internal readonly ILogger logger; - private readonly List plugins = new(); - private readonly List stagedPlugins = new(); + private static PackedPluginProvider packedPluginProvider = default!; + + private readonly List plugins = []; + private readonly List stagedPlugins = []; + private readonly List acceptedKeys = []; + private readonly IServiceProvider serverProvider; private readonly IServer server; private readonly CommandHandler commandHandler; private readonly IPluginRegistry pluginRegistry; private readonly IServiceCollection pluginServiceDescriptors = new ServiceCollection(); + + public ImmutableArray AcceptedKeys => acceptedKeys.ToImmutableArray(); /// /// List of all loaded plugins. @@ -56,9 +63,7 @@ public PluginManager(IServiceProvider serverProvider, IServer server, this.serverProvider = serverProvider; this.pluginRegistry = new PluginRegistry(this, eventDispatcher, commandHandler, logger); - PluginProviderSelector.RemotePluginProvider = new RemotePluginProvider(logger); - PluginProviderSelector.UncompiledPluginProvider = new UncompiledPluginProvider(logger); - PluginProviderSelector.CompiledPluginProvider = new CompiledPluginProvider(logger); + packedPluginProvider = new(this, logger); ConfigureInitialServices(env); @@ -94,16 +99,9 @@ private void ConfigureInitialServices(IServerEnvironment env) /// Loaded plugin. If loading failed, property will be null. public async Task LoadPluginAsync(string path) { - var provider = PluginProviderSelector.GetPluginProvider(path); - if (provider is null) - { - logger.LogError("Couldn't load plugin from path '{path}'", path); - return null; - } - try { - PluginContainer plugin = await provider.GetPluginAsync(path).ConfigureAwait(false); + PluginContainer plugin = await packedPluginProvider.GetPluginAsync(path).ConfigureAwait(false); return HandlePlugin(plugin); } diff --git a/Obsidian/Plugins/PluginProviders/CompiledPluginProvider.cs b/Obsidian/Plugins/PluginProviders/CompiledPluginProvider.cs deleted file mode 100644 index 4e2ad8039..000000000 --- a/Obsidian/Plugins/PluginProviders/CompiledPluginProvider.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Microsoft.Extensions.Logging; -using Obsidian.API.Plugins; -using System.IO; -using System.Reflection; -using System.Text.Json; - -namespace Obsidian.Plugins.PluginProviders; - -public sealed class CompiledPluginProvider(ILogger logger) : IPluginProvider -{ - private readonly ILogger logger = logger; - - public async Task GetPluginAsync(string path) - { - var loadContext = new PluginLoadContext(Path.GetFileNameWithoutExtension(path) + "LoadContext", path); - using var pluginStream = new FileStream(path, FileMode.Open); - - var assembly = loadContext.LoadFromStream(pluginStream); - - return await HandlePluginAsync(loadContext, assembly, path); - } - - internal async Task HandlePluginAsync(PluginLoadContext loadContext, Assembly assembly, string path) - { - Type? pluginType = assembly.GetTypes().FirstOrDefault(type => type.IsSubclassOf(typeof(PluginBase))); - - PluginBase? plugin; - if (pluginType == null || pluginType.GetConstructor([]) == null) - { - plugin = default; - logger.LogError("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); - return new PluginContainer(new PluginInfo(Path.GetFileNameWithoutExtension(path)), path); - } - else - { - logger.LogInformation("Creating plugin instance..."); - plugin = (PluginBase)Activator.CreateInstance(pluginType)!; - } - - string name = assembly.GetName().Name!; - using var pluginInfoStream = assembly.GetManifestResourceStream($"{name}.plugin.json") - ?? throw new InvalidOperationException($"Failed to find embedded plugin.json file for {name}"); - - var info = await pluginInfoStream.FromJsonAsync() ?? throw new JsonException($"Couldn't deserialize plugin.json from {name}"); - - return new PluginContainer(plugin, info, assembly, loadContext, path); - } -} diff --git a/Obsidian/Plugins/PluginProviders/IPluginProvider.cs b/Obsidian/Plugins/PluginProviders/IPluginProvider.cs deleted file mode 100644 index 54896474f..000000000 --- a/Obsidian/Plugins/PluginProviders/IPluginProvider.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Microsoft.Extensions.Logging; - -namespace Obsidian.Plugins.PluginProviders; - -public interface IPluginProvider -{ - public Task GetPluginAsync(string path); -} diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs new file mode 100644 index 000000000..66301ce48 --- /dev/null +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -0,0 +1,151 @@ +using Microsoft.Extensions.Logging; +using Obsidian.API.Plugins; +using Org.BouncyCastle.Crypto; +using System.Buffers; +using System.IO; +using System.IO.Compression; +using System.Reflection; +using System.Runtime.Loader; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace Obsidian.Plugins.PluginProviders; +public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger logger) +{ + private readonly PluginManager pluginManager = pluginManager; + private readonly ILogger logger = logger; + + public async Task GetPluginAsync(string path) + { + await using var fs = new FileStream(path, FileMode.Open); + using var reader = new BinaryReader(fs); + + var header = Encoding.ASCII.GetString(reader.ReadBytes(4)); + if (header != "OBBY") + throw new InvalidOperationException("Plugin file does not begin with the proper header."); + + //TODO save api version somewhere + var apiVersion = reader.ReadString(); + + var hash = reader.ReadBytes(20); + var signature = reader.ReadBytes(256); + var dataLength = reader.ReadInt32(); + + var curPos = fs.Position; + + using (var sha1 = SHA1.Create()) + { + var verifyHash = await sha1.ComputeHashAsync(fs); + + if (!verifyHash.SequenceEqual(hash)) + throw new InvalidDataException("File integrity does not match specified hash."); + } + + fs.Position = curPos; + + var pluginAssembly = reader.ReadString(); + var pluginVersion = reader.ReadString(); + + var loadContext = new PluginLoadContext(pluginAssembly); + + var entries = new PluginFileEntry[reader.ReadInt32()]; + + var offset = 0; + for (int i = 0; i < entries.Length; i++) + { + var entry = new PluginFileEntry() + { + FullName = reader.ReadString(), + Length = reader.ReadInt32(), + CompressedLength = reader.ReadInt32(), + Offset = offset, + }; + + entries[i] = entry; + + offset += entry.CompressedLength; + } + + var startPos = (int)fs.Position; + foreach (var entry in entries) + entry.Offset += startPos; + + foreach (var entry in entries) + { + byte[] actualBytes; + if (entry.Length != entry.CompressedLength) + { + var mem = new byte[entry.CompressedLength]; + + var compressedBytesRead = await fs.ReadAsync(mem); + + if (compressedBytesRead != entry.CompressedLength) + throw new DataLengthException(); + + await using var ms = new MemoryStream(mem); + await using var ds = new DeflateStream(ms, CompressionMode.Decompress); + + await using var deflatedData = new MemoryStream(); + + await ds.CopyToAsync(deflatedData); + + if (deflatedData.Length != entry.Length) + throw new DataLengthException(); + + actualBytes = deflatedData.ToArray(); + } + else + { + actualBytes = new byte[entry.Length]; + await fs.ReadAsync(actualBytes); + } + + var name = Path.GetFileNameWithoutExtension(entry.FullName); + //Don't load this assembly wait + if (name == pluginAssembly) + continue; + + //TODO LOAD OTHER FILES SOMEWHERE + if (entry.FullName.EndsWith(".dll")) + loadContext.LoadAssembly(actualBytes); + } + + var mainPluginEntry = entries.First(x => x.FullName.EndsWith($"{pluginAssembly}.dll")); + var mainPluginPbdEntry = entries.First(x => x.FullName.EndsWith($"{pluginAssembly}.pdb")); + + var mainAssembly = loadContext.LoadAssembly(await mainPluginEntry.GetDataAsync(fs), await mainPluginPbdEntry.GetDataAsync(fs)); + + if (mainAssembly is null) + throw new InvalidOperationException("Failed to find main assembly"); + + return await HandlePluginAsync(loadContext, mainAssembly, path, entries); + } + + internal async Task HandlePluginAsync(PluginLoadContext loadContext, Assembly assembly, + string path, IEnumerable entries) + { + Type? pluginType = assembly.GetTypes().FirstOrDefault(type => type.IsSubclassOf(typeof(PluginBase))); + + PluginBase? plugin; + if (pluginType == null || pluginType.GetConstructor([]) == null) + { + plugin = default; + logger.LogError("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); + + throw new InvalidOperationException("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); + } + + logger.LogInformation("Creating plugin instance..."); + plugin = (PluginBase)Activator.CreateInstance(pluginType)!; + + string name = assembly.GetName().Name!; + using var pluginInfoStream = assembly.GetManifestResourceStream($"{name}.plugin.json") + ?? throw new InvalidOperationException($"Failed to find embedded plugin.json file for {name}"); + + var info = await pluginInfoStream.FromJsonAsync() ?? + throw new JsonException($"Couldn't deserialize plugin.json from {name}"); + + return new PluginContainer(plugin, info, assembly, loadContext, path, entries); + } +} diff --git a/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs b/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs index e0afe50e2..e52d79bb9 100644 --- a/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs +++ b/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs @@ -1,34 +1,29 @@ -using System.Reflection; +using System.IO; +using System.Reflection; using System.Runtime.Loader; namespace Obsidian.Plugins.PluginProviders; -internal sealed class PluginLoadContext : AssemblyLoadContext +internal sealed class PluginLoadContext(string name) : AssemblyLoadContext(name: name, isCollectible: true) { - private readonly AssemblyDependencyResolver resolver; + public List Dependencies { get; } = []; - public PluginLoadContext(string name) : base(name: name, isCollectible: true) + public Assembly? LoadAssembly(byte[] mainBytes, byte[]? pbdBytes = null) { - } + using var mainStream = new MemoryStream(mainBytes, false); + using var pbdStream = pbdBytes != null ? new MemoryStream(pbdBytes, false) : null; - public PluginLoadContext(string name, string path) : base(name: name, isCollectible: true) - { - resolver = new AssemblyDependencyResolver(path); + var asm = this.LoadFromStream(mainStream, pbdStream); + + return asm; } + public void AddDependency(PluginLoadContext context) => this.Dependencies.Add(context); + protected override Assembly? Load(AssemblyName assemblyName) { - string assemblyPath = resolver?.ResolveAssemblyToPath(assemblyName); - if (assemblyPath != null) - { - return LoadFromAssemblyPath(assemblyPath); - } + var assembly = this.Assemblies.FirstOrDefault(x => x.GetName() == assemblyName); - return null; - } - - protected override IntPtr LoadUnmanagedDll(string unmanagedDllName) - { - return IntPtr.Zero; + return assembly ?? this.Dependencies.Select(x => x.Load(assemblyName)).FirstOrDefault(x => x != null); } } diff --git a/Obsidian/Plugins/PluginProviders/PluginProviderSelector.cs b/Obsidian/Plugins/PluginProviders/PluginProviderSelector.cs deleted file mode 100644 index 52bbe6f38..000000000 --- a/Obsidian/Plugins/PluginProviders/PluginProviderSelector.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.IO; - -namespace Obsidian.Plugins.PluginProviders; - -public static class PluginProviderSelector -{ - public static RemotePluginProvider RemotePluginProvider { get; internal set; } = default!; - public static UncompiledPluginProvider UncompiledPluginProvider { get; internal set; } = default!; - public static CompiledPluginProvider CompiledPluginProvider { get; internal set; } = default!; - - - public static IPluginProvider? GetPluginProvider(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - - if (IsUrl(path)) - { - return RemotePluginProvider; - } - else - { - var fileExtension = Path.GetExtension(path); - return fileExtension switch - { - ".cs" => UncompiledPluginProvider, - ".dll" => CompiledPluginProvider, - _ => null - }; - } - } - - private static bool IsUrl(string path) => - Uri.TryCreate(path, UriKind.Absolute, out var url) && (url.Scheme == Uri.UriSchemeHttp || url.Scheme == Uri.UriSchemeHttps); -} diff --git a/Obsidian/Plugins/PluginProviders/RemotePluginProvider.cs b/Obsidian/Plugins/PluginProviders/RemotePluginProvider.cs deleted file mode 100644 index d4ba25f40..000000000 --- a/Obsidian/Plugins/PluginProviders/RemotePluginProvider.cs +++ /dev/null @@ -1,238 +0,0 @@ -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.Emit; -using Microsoft.CodeAnalysis.Text; -using Microsoft.Extensions.Logging; -using System.IO; -using System.Net.Http; -using System.Text.Json; -using System.Xml; - -namespace Obsidian.Plugins.PluginProviders; - -public class RemotePluginProvider(ILogger logger) : IPluginProvider -{ - private static HttpClient client; - - static RemotePluginProvider() - { - client = Globals.HttpClient; - client.DefaultRequestHeaders.Add("User-Agent", "ObsidianServer"); - } - - public async Task GetPluginAsync(string path) - { - if (!Uri.TryCreate(path, UriKind.Absolute, out Uri url)) - return Failed("'unknown'", path, logger, "Provided path is not a valid url"); - - var repository = ParseUrl(url.AbsolutePath); - - return url.Host switch - { - "www.github.com" => await LoadGithubPluginAsync(path, repository), - "gist.github.com" => await LoadGistPluginAsync(path, repository), - _ => Failed("'unknown'", path, logger, $"Remote source {url.Host} is not supported") - }; - } - - private async Task LoadGithubPluginAsync(string path, (string owner, string name) repository) - { - string branch = "main"; - JsonDocument scan = await ScanDirectoryAsync(repository.owner, repository.name, branch); - if (scan is null || scan.RootElement.TryGetProperty("message", out _)) - { - scan?.Dispose(); - branch = "master"; - scan = await ScanDirectoryAsync(repository.owner, repository.name, branch); - if (scan is null) - { - return Failed(repository.name, path, logger, "GitHub servers are not accessible"); - } - else if (scan.RootElement.TryGetProperty("message", out _)) - { - scan.Dispose(); - return Failed(repository.name, path, logger, "Repository is missing master/main tree"); - } - } - - JsonDocument obsidianFile = default; - JsonElement tree = scan.RootElement.GetProperty("tree"); - foreach (JsonElement item in tree.EnumerateArray()) - { - string itemPath = item.GetProperty("path").GetString(); - if (itemPath.EndsWith(".obsidian")) - { - if (obsidianFile != default) - return Failed(repository.name, path, logger, "Repository contains multiple obsidian files"); - - obsidianFile = await GetJsonFileAsync(repository.owner, repository.name, branch, itemPath); - } - } - - (XmlDocument csproj, string csprojPath) = await GetCsprojAsync(obsidianFile, tree, repository, branch); - - if (csproj == null) - return Failed(repository.name, path, logger, "Project not found inside plugin repository"); - - string projectPath = Path.GetDirectoryName(csprojPath).Replace('\\', '/'); - - var fileStreams = new List(); - foreach (JsonElement item in tree.EnumerateArray()) - { - string itemPath = item.GetProperty("path").GetString(); - string itemType = item.GetProperty("type").GetString(); - if (itemPath.StartsWith(projectPath) && itemPath.EndsWith(".cs") && itemType == "blob") - { - fileStreams.Add(await GetFileStreamAsync(repository.owner, repository.name, branch, itemPath)); - } - } - - scan.Dispose(); - obsidianFile?.Dispose(); - - return await CompilePluginFilesAsync(repository, path, fileStreams); - } - - private async Task LoadGistPluginAsync(string path, (string owner, string name) repository) - { - Stream stream = await GetFileStreamAsync(repository.owner, repository.name); - return await CompilePluginFilesAsync(repository, path, new[] { stream }); - } - - private async Task CompilePluginFilesAsync((string owner, string name) repository, string path, IEnumerable fileStreams) - { - var syntaxTrees = fileStreams.Select(fileStream => CSharpSyntaxTree.ParseText(SourceText.From(fileStream))); - var compilation = CSharpCompilation.Create(repository.name, - syntaxTrees, - UncompiledPluginProvider.MetadataReferences, - UncompiledPluginProvider.CompilationOptions); - using var memoryStream = new MemoryStream(); - EmitResult compilationResult = compilation.Emit(memoryStream); - if (!compilationResult.Success) - { - if (logger != null) - { - foreach (var diagnostic in compilationResult.Diagnostics) - { - if (diagnostic.Severity != DiagnosticSeverity.Error || diagnostic.IsWarningAsError) - continue; - - logger.LogError("Compilation failed: {Location} {Message}", diagnostic.Location, diagnostic.GetMessage()); - } - } - - return Failed(repository.name, path); - } - else - { - memoryStream.Seek(0, SeekOrigin.Begin); - - var loadContext = new PluginLoadContext(repository.name + "LoadContext"); - var assembly = loadContext.LoadFromStream(memoryStream); - memoryStream.Dispose(); - return await PluginProviderSelector.CompiledPluginProvider.HandlePluginAsync(loadContext, assembly, path); - } - } - - private async Task<(XmlDocument xml, string path)> GetCsprojAsync(JsonDocument obsidianFile, JsonElement tree, (string owner, string name) repository, string treeName) - { - JsonElement? csproj = null; - string target = null; - JsonElement targetElement = default; - if (obsidianFile?.RootElement.TryGetProperty("target", out targetElement) != false) - target = targetElement.GetString() + ".csproj"; - if (target != null) - { - csproj = tree.EnumerateArray().FirstOrDefault(item => - { - string str = item.GetProperty("path").GetString(); - int slashIndex = str.LastIndexOf('/'); - return slashIndex != -1 ? str[(slashIndex + 1)..] == target : str == target; - }); - } - else - { - foreach (var item in tree.EnumerateArray()) - { - string str = item.GetProperty("path").GetString(); - int slashIndex = str.LastIndexOf('/'); - if (slashIndex != -1 ? str[(slashIndex + 1)..] == target : str == target) - { - csproj = item; - break; - } - } - } - - if (!csproj.HasValue) - return default; - - string csprojPath = csproj.Value.GetProperty("path").GetString(); - return (await GetXmlFileAsync(repository.owner, repository.name, treeName, csprojPath), csprojPath); - } - - private async Task ScanDirectoryAsync(string owner, string name, string tree) - { - await using var stream = await GetFileStreamAsync($"https://api.github.com/repos/{owner}/{name}/git/trees/{tree}?recursive=1"); - if (stream == null) - return null; - return await JsonDocument.ParseAsync(stream); - } - - private async Task GetJsonFileAsync(string owner, string name, string tree, string path) - { - await using var stream = await GetFileStreamAsync($"https://www.github.com/{owner}/{name}/raw/{tree}/{path}"); - if (stream == null) - return null; - return await JsonDocument.ParseAsync(stream); - } - - private async Task GetXmlFileAsync(string owner, string name, string tree, string path) - { - await using var stream = await GetFileStreamAsync($"https://www.github.com/{owner}/{name}/raw/{tree}/{path}"); - if (stream == null) - return null; - - XmlDocument xml = new XmlDocument(); - xml.Load(stream); - return xml; - } - - private async Task GetFileStreamAsync(string owner, string name, string tree, string path) - { - var response = await client.GetAsync($"https://www.github.com/{owner}/{name}/raw/{tree}/{path}"); - if (!response.IsSuccessStatusCode) - return null; - return await response.Content.ReadAsStreamAsync(); - } - - private async Task GetFileStreamAsync(string owner, string hash) - { - var response = await client.GetAsync($"https://gist.githubusercontent.com/{owner}/{hash}/raw"); - if (!response.IsSuccessStatusCode) - return null; - return await response.Content.ReadAsStreamAsync(); - } - - private async Task GetFileStreamAsync(string url) - { - var response = await client.GetAsync(url); - if (!response.IsSuccessStatusCode) - return null; - - return await response.Content.ReadAsStreamAsync(); - } - - private (string owner, string name) ParseUrl(string url) - { - int nameIndex = url.LastIndexOf('/'); - int ownerIndex = url.LastIndexOf('/', nameIndex - 1); - return (url.Substring(ownerIndex + 1, nameIndex - ownerIndex - 1), url[(nameIndex + 1)..]); - } - - private static PluginContainer Failed(string name, string path, ILogger logger = default, string reason = null) - { - logger?.LogError($"Loading plugin {name} failed with reason: {reason}"); - return new PluginContainer(new PluginInfo(name), path); - } -} diff --git a/Obsidian/Plugins/PluginProviders/UncompiledPluginProvider.cs b/Obsidian/Plugins/PluginProviders/UncompiledPluginProvider.cs deleted file mode 100644 index f75279331..000000000 --- a/Obsidian/Plugins/PluginProviders/UncompiledPluginProvider.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.Emit; -using Microsoft.CodeAnalysis.Text; -using Microsoft.Extensions.Logging; -using System.IO; - -namespace Obsidian.Plugins.PluginProviders; - -public class UncompiledPluginProvider(ILogger logger) : IPluginProvider -{ - private static WeakReference _metadataReferences = new WeakReference(null); - internal static PortableExecutableReference[] MetadataReferences - { - get - { - if (_metadataReferences.TryGetTarget(out var value) && value != null) - return value; - var references = GetPortableExecutableReferences(); - _metadataReferences.SetTarget(references); - return references; - } - } - internal static CSharpCompilationOptions CompilationOptions { get; set; } - - static UncompiledPluginProvider() - { - CompilationOptions = new CSharpCompilationOptions(outputKind: OutputKind.DynamicallyLinkedLibrary, -#if RELEASE - optimizationLevel: OptimizationLevel.Release); -#elif DEBUG - optimizationLevel: OptimizationLevel.Debug); -#endif - } - - public async Task GetPluginAsync(string path) - { - string name = Path.GetFileNameWithoutExtension(path); - - FileStream fileStream; - try - { - fileStream = File.OpenRead(path); - } - catch - { - logger.LogError("Reloading '{filePath}' failed, file is not accessible.", Path.GetFileName(path)); - return new PluginContainer(new PluginInfo(name), name); - } - SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(SourceText.From(fileStream)); - fileStream.Dispose(); - var compilation = CSharpCompilation.Create(name, - new[] { syntaxTree }, - MetadataReferences, - CompilationOptions); - using var memoryStream = new MemoryStream(); - EmitResult emitResult = compilation.Emit(memoryStream); - - if (!emitResult.Success) - { - if (logger != null) - { - foreach (var diagnostic in emitResult.Diagnostics) - { - if (diagnostic.Severity != DiagnosticSeverity.Error || diagnostic.IsWarningAsError) - continue; - - logger.LogError("Compilation failed: {diagnosticLocation} {diagnosticMessage}", diagnostic.Location, diagnostic.GetMessage()); - } - } - - return new PluginContainer(new PluginInfo(name), name); - } - else - { - memoryStream.Seek(0, SeekOrigin.Begin); - - var loadContext = new PluginLoadContext(name + "LoadContext", path); - var assembly = loadContext.LoadFromStream(memoryStream); - return await PluginProviderSelector.CompiledPluginProvider.HandlePluginAsync(loadContext, assembly, path); - } - } - - private static PortableExecutableReference[] GetPortableExecutableReferences() - { - var trustedAssembliesPaths = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")).Split(Path.PathSeparator); - return trustedAssembliesPaths.Select(reference => MetadataReference.CreateFromFile(reference)).ToArray(); - } -} diff --git a/Obsidian/Server.cs b/Obsidian/Server.cs index d4f67ef67..a51517403 100644 --- a/Obsidian/Server.cs +++ b/Obsidian/Server.cs @@ -256,7 +256,7 @@ public async Task RunAsync() Directory.CreateDirectory("plugins"); - PluginManager.DirectoryWatcher.Filters = new[] { ".cs", ".dll" }; + PluginManager.DirectoryWatcher.Filters = [".obby"]; PluginManager.DirectoryWatcher.Watch("plugins"); await Task.WhenAll(Configuration.DownloadPlugins.Select(path => PluginManager.LoadPluginAsync(path))); diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 0e6520526..831a602a4 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -9,9 +9,10 @@ true false - SamplePlugin + SamplePlugin 1.0.0.0 1.0.0 + bin/$(Configuration)/ @@ -19,10 +20,11 @@ + runtime - + @@ -32,7 +34,15 @@ - - + + + + + + + From 5e4341efb3b7a430a956669cc211152441ac8746 Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 29 Feb 2024 19:46:48 -0500 Subject: [PATCH 04/21] Update SamplePlugin.csproj --- SamplePlugin/SamplePlugin.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 831a602a4..da043a86c 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -24,7 +24,7 @@ runtime - + @@ -34,14 +34,14 @@ - + From 95b5bbd22f0b5f86639b6fc40d26d1b58f3377c6 Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 29 Feb 2024 22:13:10 -0500 Subject: [PATCH 05/21] Properly dispose of plugin and unload context --- Obsidian/Plugins/PluginContainer.cs | 77 +++++++++++-------- Obsidian/Plugins/PluginFileEntry.cs | 6 +- Obsidian/Plugins/PluginManager.cs | 8 +- .../PluginProviders/PackedPluginProvider.cs | 60 +++++++++++---- SamplePlugin/SamplePlugin.csproj | 12 +-- 5 files changed, 104 insertions(+), 59 deletions(-) diff --git a/Obsidian/Plugins/PluginContainer.cs b/Obsidian/Plugins/PluginContainer.cs index 469fa27b2..1812e3bcf 100644 --- a/Obsidian/Plugins/PluginContainer.cs +++ b/Obsidian/Plugins/PluginContainer.cs @@ -1,7 +1,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; +using System.Collections.Frozen; using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Reflection; using System.Runtime.Loader; @@ -10,34 +12,39 @@ namespace Obsidian.Plugins; public sealed class PluginContainer : IDisposable { - private Type? pluginType; - + private Type PluginType => this.Plugin.GetType(); public IServiceScope ServiceScope { get; internal set; } = default!; + public PluginInfo Info { get; private set; } = default!; + + [AllowNull] + public PluginBase Plugin { get; internal set; } = default!; - public PluginBase? Plugin { get; private set; } - public PluginInfo Info { get; } + [AllowNull] + public AssemblyLoadContext LoadContext { get; internal set; } = default!; - public AssemblyLoadContext? LoadContext { get; private set; } - public Assembly PluginAssembly { get; } = default!; + [AllowNull] + public Assembly PluginAssembly { get; internal set; } = default!; - public string Source { get; internal set; } = default!; + [AllowNull] + public FrozenDictionary FileEntries { get; internal set; } = default!; + public required string Source { get; set; } + public bool HasDependencies { get; private set; } = true; public bool IsReady => HasDependencies; public bool Loaded { get; internal set; } - public ImmutableArray FileEntries { get; } + ~PluginContainer() + { + this.Dispose(false); + } - public PluginContainer(PluginBase plugin, PluginInfo info, Assembly assembly, AssemblyLoadContext loadContext, - string source, IEnumerable fileEntries) + internal async Task InitializeAsync() { - Plugin = plugin; - Info = info; - LoadContext = loadContext; - Source = source; - PluginAssembly = assembly; - pluginType = plugin.GetType(); - Plugin.Info = Info; - this.FileEntries = fileEntries.ToImmutableArray(); + var pluginJsonData = await this.GetFileDataAsync("plugin.json") ?? throw new InvalidOperationException("Failed to find plugin.json"); + + await using var pluginInfoStream = new MemoryStream(pluginJsonData, false); + this.Info = await pluginInfoStream.FromJsonAsync() ?? throw new NullReferenceException("Failed to deserialize plugin.json"); + this.Plugin.Info = this.Info; } /// @@ -47,19 +54,13 @@ public PluginContainer(PluginBase plugin, PluginInfo info, Assembly assembly, As /// Null if the file is not found or the byte array of the file. public async Task GetFileDataAsync(string fileName) { - var fileEntry = this.FileEntries.FirstOrDefault(x => Path.GetFileName(x.FullName) == fileName); + var fileEntry = this.FileEntries.GetValueOrDefault(fileName); if (fileEntry is null) return null; - await using var fs = new FileStream(this.Source, FileMode.Open); - - fs.Seek(fileEntry.Offset, SeekOrigin.Begin); - - var data = new byte[fileEntry.CompressedLength]; + await using var fs = new FileStream(this.Source, FileMode.Open, FileAccess.Read, FileShare.Read); - await fs.ReadAsync(data); - - return data; + return await fileEntry.GetDataAsync(fs); } /// @@ -67,7 +68,7 @@ public PluginContainer(PluginBase plugin, PluginInfo info, Assembly assembly, As /// public void InjectServices(ILogger? logger, object? target = null) { - var properties = target is null ? this.pluginType!.WithInjectAttribute() : target.GetType().WithInjectAttribute(); + var properties = target is null ? this.PluginType!.WithInjectAttribute() : target.GetType().WithInjectAttribute(); target ??= this.Plugin; @@ -89,10 +90,22 @@ public void InjectServices(ILogger? logger, object? target = null) public void Dispose() { - Plugin = null; - LoadContext = null; - pluginType = null; + this.Dispose(true); + + GC.SuppressFinalize(this); + } - this.ServiceScope.Dispose(); + private void Dispose(bool disposing) + { + if(this.ServiceScope != null) + this.ServiceScope.Dispose(); + + if (disposing) + { + this.PluginAssembly = null; + this.LoadContext = null; + this.Plugin = null; + this.FileEntries = null; + } } } diff --git a/Obsidian/Plugins/PluginFileEntry.cs b/Obsidian/Plugins/PluginFileEntry.cs index 53fd2863a..05ca27341 100644 --- a/Obsidian/Plugins/PluginFileEntry.cs +++ b/Obsidian/Plugins/PluginFileEntry.cs @@ -6,7 +6,7 @@ namespace Obsidian.Plugins; public sealed class PluginFileEntry { - public required string FullName { get; init; } + public required string Name { get; init; } public required int Length { get; init; } @@ -16,7 +16,7 @@ public sealed class PluginFileEntry public bool IsCompressed => CompressedLength < Length; - public async Task GetDataAsync(FileStream packedPluginFile) + internal async Task GetDataAsync(FileStream packedPluginFile) { packedPluginFile.Seek(this.Offset, SeekOrigin.Begin); @@ -32,7 +32,7 @@ public async Task GetDataAsync(FileStream packedPluginFile) if (await packedPluginFile.ReadAsync(compressedData.AsMemory(0, this.CompressedLength)) != this.CompressedLength) throw new DataLengthException(); - await using var ms = new MemoryStream(compressedData); + await using var ms = new MemoryStream(compressedData, false); await using var ds = new DeflateStream(ms, CompressionMode.Decompress); await using var outStream = new MemoryStream(); diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index e36767ac9..9cf05d6f9 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -164,6 +164,8 @@ private void ConfigureInitialServices(IServerEnvironment env) /// public async Task UnloadPluginAsync(PluginContainer pluginContainer) { + this.logger.LogInformation("Unloading plugin..."); + bool removed = false; lock (plugins) { @@ -189,10 +191,12 @@ public async Task UnloadPluginAsync(PluginContainer pluginContainer) logger.LogError(ex, "Unhandled exception occured when disposing {pluginName}", pluginContainer.Info.Name); } - pluginContainer.LoadContext.Unload(); - pluginContainer.LoadContext.Unloading += _ => logger.LogInformation("Finished unloading {pluginName} plugin", pluginContainer.Info.Name); + var loadContext = pluginContainer.LoadContext; pluginContainer.Dispose(); + + loadContext.Unloading += _ => logger.LogInformation("Finished unloading {pluginName} plugin", pluginContainer.Info.Name); + loadContext.Unload(); } public void ServerReady() diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index 66301ce48..b7d32708e 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -2,6 +2,7 @@ using Obsidian.API.Plugins; using Org.BouncyCastle.Crypto; using System.Buffers; +using System.Collections.Frozen; using System.IO; using System.IO.Compression; using System.Reflection; @@ -49,29 +50,31 @@ public async Task GetPluginAsync(string path) var loadContext = new PluginLoadContext(pluginAssembly); - var entries = new PluginFileEntry[reader.ReadInt32()]; + var entryCount = reader.ReadInt32(); + var entries = new Dictionary(entryCount); var offset = 0; - for (int i = 0; i < entries.Length; i++) + for (int i = 0; i < entryCount; i++) { var entry = new PluginFileEntry() { - FullName = reader.ReadString(), + Name = reader.ReadString(), Length = reader.ReadInt32(), CompressedLength = reader.ReadInt32(), Offset = offset, }; - entries[i] = entry; + entries.Add(entry.Name, entry); offset += entry.CompressedLength; } var startPos = (int)fs.Position; - foreach (var entry in entries) + foreach (var (_, entry) in entries) entry.Offset += startPos; - foreach (var entry in entries) + var libsWithSymbols = new List(); + foreach (var (_, entry) in entries) { byte[] actualBytes; if (entry.Length != entry.CompressedLength) @@ -101,29 +104,48 @@ public async Task GetPluginAsync(string path) await fs.ReadAsync(actualBytes); } - var name = Path.GetFileNameWithoutExtension(entry.FullName); + var name = Path.GetFileNameWithoutExtension(entry.Name); //Don't load this assembly wait if (name == pluginAssembly) continue; //TODO LOAD OTHER FILES SOMEWHERE - if (entry.FullName.EndsWith(".dll")) + if (entry.Name.EndsWith(".dll")) + { + if(entries.ContainsKey(entry.Name.Replace(".dll", ".pdb"))) + { + //Library has debug symbols load in last + libsWithSymbols.Add(entry.Name.Replace(".dll", ".pdb")); + continue; + } + loadContext.LoadAssembly(actualBytes); + } } - var mainPluginEntry = entries.First(x => x.FullName.EndsWith($"{pluginAssembly}.dll")); - var mainPluginPbdEntry = entries.First(x => x.FullName.EndsWith($"{pluginAssembly}.pdb")); + foreach(var lib in libsWithSymbols) + { + var mainLib = await entries[$"{lib}.dll"].GetDataAsync(fs); + var libSymbols = await entries[$"{lib}.pdb"].GetDataAsync(fs); + + loadContext.LoadAssembly(mainLib, libSymbols); + } + + var mainPluginEntry = entries[$"{pluginAssembly}.dll"]; + var mainPluginPbdEntry = entries[$"{pluginAssembly}.pdb"]; var mainAssembly = loadContext.LoadAssembly(await mainPluginEntry.GetDataAsync(fs), await mainPluginPbdEntry.GetDataAsync(fs)); if (mainAssembly is null) throw new InvalidOperationException("Failed to find main assembly"); + await fs.DisposeAsync(); + return await HandlePluginAsync(loadContext, mainAssembly, path, entries); } internal async Task HandlePluginAsync(PluginLoadContext loadContext, Assembly assembly, - string path, IEnumerable entries) + string path, Dictionary entries) { Type? pluginType = assembly.GetTypes().FirstOrDefault(type => type.IsSubclassOf(typeof(PluginBase))); @@ -139,13 +161,17 @@ internal async Task HandlePluginAsync(PluginLoadContext loadCon logger.LogInformation("Creating plugin instance..."); plugin = (PluginBase)Activator.CreateInstance(pluginType)!; - string name = assembly.GetName().Name!; - using var pluginInfoStream = assembly.GetManifestResourceStream($"{name}.plugin.json") - ?? throw new InvalidOperationException($"Failed to find embedded plugin.json file for {name}"); + var pluginContainer = new PluginContainer + { + Plugin = plugin, + FileEntries = entries.ToFrozenDictionary(), + LoadContext = loadContext, + PluginAssembly = assembly, + Source = path + }; - var info = await pluginInfoStream.FromJsonAsync() ?? - throw new JsonException($"Couldn't deserialize plugin.json from {name}"); + await pluginContainer.InitializeAsync(); - return new PluginContainer(plugin, info, assembly, loadContext, path, entries); + return pluginContainer; } } diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index da043a86c..6fe46c5f1 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -15,16 +15,12 @@ bin/$(Configuration)/ - - - - runtime - + @@ -34,6 +30,12 @@ + + + PreserveNewest + + + From e25888e99dd070dfce4439ff690480c396a8b770 Mon Sep 17 00:00:00 2001 From: Tides Date: Sat, 2 Mar 2024 21:24:40 -0500 Subject: [PATCH 08/21] Checkpoint --- Obsidian.API/Plugins/IPluginInfo.cs | 17 ++ Obsidian/Plugins/DirectoryWatcher.cs | 14 +- Obsidian/Plugins/PluginContainer.cs | 36 ++-- Obsidian/Plugins/PluginFileEntry.cs | 29 +-- Obsidian/Plugins/PluginInfo.cs | 7 - Obsidian/Plugins/PluginManager.cs | 159 ++++++++------- .../PluginProviders/PackedPluginProvider.cs | 184 ++++++++++-------- .../PluginProviders/PluginLoadContext.cs | 4 +- Obsidian/Server.cs | 5 +- Obsidian/Utilities/Extensions.cs | 7 +- SamplePlugin/SamplePlugin.csproj | 5 +- 11 files changed, 257 insertions(+), 210 deletions(-) diff --git a/Obsidian.API/Plugins/IPluginInfo.cs b/Obsidian.API/Plugins/IPluginInfo.cs index aae5d2259..91ac57217 100644 --- a/Obsidian.API/Plugins/IPluginInfo.cs +++ b/Obsidian.API/Plugins/IPluginInfo.cs @@ -7,4 +7,21 @@ public interface IPluginInfo public string Description { get; } public string[] Authors { get; } public Uri ProjectUrl { get; } + + public PluginDependency[] Dependencies { get; } +} + +public readonly struct PluginDependency +{ + public required string Name { get; init; } + + public required string Version { get; init; } + + public DependencyPriority Priority { get; init; } +} + +public enum DependencyPriority +{ + Soft, + Hard } diff --git a/Obsidian/Plugins/DirectoryWatcher.cs b/Obsidian/Plugins/DirectoryWatcher.cs index 5148a835e..df87fdc5e 100644 --- a/Obsidian/Plugins/DirectoryWatcher.cs +++ b/Obsidian/Plugins/DirectoryWatcher.cs @@ -7,9 +7,9 @@ public sealed class DirectoryWatcher : IDisposable private string[] _filters = []; public string[] Filters { get => _filters; set => _filters = value ?? []; } - public event Action FileChanged; - public event Action FileRenamed; - public event Action FileDeleted; + public event Action FileChanged = default!; + public event Action FileRenamed = default!; + public event Action FileDeleted = default!; private readonly Dictionary watchers = new(); private readonly Dictionary updateTimestamps = new(); @@ -21,12 +21,6 @@ public void Watch(string path) if (!Directory.Exists(path)) throw new DirectoryNotFoundException(path); - foreach (var file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) - { - if (TestFilter(file)) - FileChanged?.Invoke(file); - } - lock (watchers) { if (watchers.ContainsKey(path)) @@ -56,7 +50,7 @@ public void Unwatch(string path) { lock (watchers) { - if (!watchers.TryGetValue(path, out FileSystemWatcher watcher)) + if (!watchers.TryGetValue(path, out FileSystemWatcher? watcher)) throw new KeyNotFoundException(); watcher.Created -= OnFileUpdated; diff --git a/Obsidian/Plugins/PluginContainer.cs b/Obsidian/Plugins/PluginContainer.cs index 4756fd139..6b22bcf37 100644 --- a/Obsidian/Plugins/PluginContainer.cs +++ b/Obsidian/Plugins/PluginContainer.cs @@ -1,17 +1,18 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; +using Obsidian.Plugins.PluginProviders; using System.Collections.Frozen; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Reflection; -using System.Runtime.Loader; namespace Obsidian.Plugins; public sealed class PluginContainer : IDisposable { + private bool initialized; + private Type PluginType => this.Plugin.GetType(); public IServiceScope ServiceScope { get; internal set; } = default!; public PluginInfo Info { get; private set; } = default!; @@ -20,7 +21,7 @@ public sealed class PluginContainer : IDisposable public PluginBase Plugin { get; internal set; } = default!; [AllowNull] - public AssemblyLoadContext LoadContext { get; internal set; } = default!; + public PluginLoadContext LoadContext { get; internal set; } = default!; [AllowNull] public Assembly PluginAssembly { get; internal set; } = default!; @@ -28,7 +29,7 @@ public sealed class PluginContainer : IDisposable [AllowNull] public FrozenDictionary FileEntries { get; internal set; } = default!; public required string Source { get; set; } - + public bool HasDependencies { get; private set; } = true; public bool IsReady => HasDependencies; public bool Loaded { get; internal set; } @@ -38,12 +39,19 @@ public sealed class PluginContainer : IDisposable this.Dispose(false); } - internal async Task InitializeAsync() + internal void Initialize() { - var pluginJsonData = await this.GetFileDataAsync("plugin.json") ?? throw new InvalidOperationException("Failed to find plugin.json"); + if (!this.initialized) + { + var pluginJsonData = this.GetFileData("plugin.json") ?? throw new InvalidOperationException("Failed to find plugin.json"); + + this.Info = pluginJsonData.FromJson() ?? throw new NullReferenceException("Failed to deserialize plugin.json"); + + this.initialized = true; + + return; + } - await using var pluginInfoStream = new MemoryStream(pluginJsonData, false); - this.Info = await pluginInfoStream.FromJsonAsync() ?? throw new NullReferenceException("Failed to deserialize plugin.json"); this.Plugin.Info = this.Info; } @@ -52,17 +60,17 @@ internal async Task InitializeAsync() /// /// The name of the file you're searching for. /// Null if the file is not found or the byte array of the file. - public async Task GetFileDataAsync(string fileName) + public byte[]? GetFileData(string fileName) { var fileEntry = this.FileEntries.GetValueOrDefault(fileName); - if (fileEntry is null) - return null; - await using var fs = new FileStream(this.Source, FileMode.Open, FileAccess.Read, FileShare.Read); - - return await fileEntry.GetDataAsync(fs); + return fileEntry?.GetData(); } + //TODO PLUGINS SHOULD USE VERSION CLASS TO SPECIFY VERSION + public bool IsDependency(string pluginName) => + this.Info.Dependencies.Any(x => x.Name == pluginName); + /// /// Inject the scoped services into /// diff --git a/Obsidian/Plugins/PluginFileEntry.cs b/Obsidian/Plugins/PluginFileEntry.cs index 05ca27341..eeaa95e6a 100644 --- a/Obsidian/Plugins/PluginFileEntry.cs +++ b/Obsidian/Plugins/PluginFileEntry.cs @@ -6,6 +6,8 @@ namespace Obsidian.Plugins; public sealed class PluginFileEntry { + internal byte[] rawData = default!; + public required string Name { get; init; } public required int Length { get; init; } @@ -16,30 +18,17 @@ public sealed class PluginFileEntry public bool IsCompressed => CompressedLength < Length; - internal async Task GetDataAsync(FileStream packedPluginFile) + internal byte[] GetData() { - packedPluginFile.Seek(this.Offset, SeekOrigin.Begin); - if (!this.IsCompressed) - { - var mem = new byte[this.Length]; - - return await packedPluginFile.ReadAsync(mem) != this.Length ? throw new DataLengthException() : mem; - } - - var compressedData = ArrayPool.Shared.Rent(this.CompressedLength); - - if (await packedPluginFile.ReadAsync(compressedData.AsMemory(0, this.CompressedLength)) != this.CompressedLength) - throw new DataLengthException(); - - await using var ms = new MemoryStream(compressedData, false); - await using var ds = new DeflateStream(ms, CompressionMode.Decompress); - await using var outStream = new MemoryStream(); + return this.rawData; - await ds.CopyToAsync(outStream); + using var ms = new MemoryStream(this.rawData, false); + using var ds = new DeflateStream(ms, CompressionMode.Decompress); + using var outStream = new MemoryStream(); - ArrayPool.Shared.Return(compressedData); + ds.CopyTo(outStream); - return outStream.ToArray(); + return outStream.Length != this.Length ? throw new DataLengthException() : outStream.ToArray(); } } diff --git a/Obsidian/Plugins/PluginInfo.cs b/Obsidian/Plugins/PluginInfo.cs index a46066887..bfbee6411 100644 --- a/Obsidian/Plugins/PluginInfo.cs +++ b/Obsidian/Plugins/PluginInfo.cs @@ -26,10 +26,3 @@ internal PluginInfo(string name) Authors = ["Unknown"]; } } - -public readonly struct PluginDependency -{ - public required string Id { get; init; } - - public required Version Version { get; init; } -} diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 8f2e0777c..d1d9e67b8 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -9,6 +9,8 @@ using Obsidian.Registries; using Obsidian.Services; using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; using System.Reflection; namespace Obsidian.Plugins; @@ -28,7 +30,7 @@ public sealed class PluginManager private readonly CommandHandler commandHandler; private readonly IPluginRegistry pluginRegistry; private readonly IServiceCollection pluginServiceDescriptors = new ServiceCollection(); - + public ImmutableArray AcceptedKeys => acceptedKeys.ToImmutableArray(); /// @@ -65,6 +67,7 @@ public PluginManager(IServiceProvider serverProvider, IServer server, ConfigureInitialServices(env); + DirectoryWatcher.Filters = [".obby"]; DirectoryWatcher.FileChanged += async (path) => { var old = plugins.FirstOrDefault(plugin => plugin.Source == path) ?? @@ -79,23 +82,35 @@ public PluginManager(IServiceProvider serverProvider, IServer server, DirectoryWatcher.FileDeleted += OnPluginSourceDeleted; } - private void ConfigureInitialServices(IServerEnvironment env) + + public async Task LoadPluginsAsync() { - this.pluginServiceDescriptors.AddLogging((builder) => + var files = Directory.GetFiles("plugins", "*.obby", SearchOption.AllDirectories); + + var waitingForDepend = new List(); + foreach (var file in files) { - builder.ClearProviders(); - builder.AddProvider(new LoggerProvider(env.Configuration.LogLevel)); - builder.SetMinimumLevel(env.Configuration.LogLevel); - }); - this.pluginServiceDescriptors.AddSingleton(x => env.Configuration); + var pluginContainer = await this.LoadPluginAsync(file); + + if (pluginContainer.Plugin is null) + waitingForDepend.Add(pluginContainer); + } + + foreach (var pluginToLoad in waitingForDepend) + { + packedPluginProvider.InitializePlugin(pluginToLoad); + this.HandlePlugin(pluginToLoad); + } + + DirectoryWatcher.Watch("plugins"); } /// /// Loads a plugin from selected path asynchronously. /// - /// Path to load the plugin from. Can point either to local DLL, C# code file or a GitHub project url. + /// Path to load the plugin from. Can point either to local OBBY or DLL. /// Loaded plugin. If loading failed, property will be null. - public async Task LoadPluginAsync(string path) + public async Task LoadPluginAsync(string path) { try { @@ -111,53 +126,6 @@ private void ConfigureInitialServices(IServerEnvironment env) } } - private PluginContainer? HandlePlugin(PluginContainer pluginContainer) - { - if (pluginContainer?.Plugin is null) - { - return pluginContainer; - } - - //Inject first wave of services (services initialized by obsidian e.x IServerConfiguration) - PluginServiceHandler.InjectServices(this.serverProvider, pluginContainer, this.logger); - - if (pluginContainer.IsReady) - { - lock (plugins) - { - plugins.Add(pluginContainer); - } - - pluginContainer.Plugin.ConfigureServices(this.pluginServiceDescriptors); - pluginContainer.Plugin.ConfigureRegistry(this.pluginRegistry); - - pluginContainer.Loaded = true; - - InvokeOnLoad(pluginContainer); - } - else - { - lock (stagedPlugins) - { - stagedPlugins.Add(pluginContainer); - } - - if (logger != null) - { - var stageMessage = new System.Text.StringBuilder(50); - stageMessage.Append($"Plugin {pluginContainer.Info.Name} staged"); - if (!pluginContainer.HasDependencies) - stageMessage.Append(", missing dependencies"); - - logger.LogWarning("{}", stageMessage.ToString()); - } - } - - logger?.LogInformation("Loading finished!"); - - return pluginContainer; - } - /// /// Will cause selected plugin to be unloaded asynchronously. /// @@ -181,7 +149,9 @@ public async Task UnloadPluginAsync(PluginContainer pluginContainer) this.commandHandler.UnregisterPluginCommands(pluginContainer); - InvokeOnUnloading(pluginContainer); + var stopwatch = Stopwatch.StartNew(); + + await pluginContainer.Plugin.OnUnloadingAsync(); try { @@ -197,7 +167,9 @@ public async Task UnloadPluginAsync(PluginContainer pluginContainer) //Dispose has to be called before the LoadContext can unload. pluginContainer.Dispose(); - loadContext.Unloading += _ => logger.LogInformation("Finished unloading {pluginName} plugin", pluginContainer.Info.Name); + stopwatch.Stop(); + + loadContext.Unloading += _ => logger.LogInformation("Finished unloading {pluginName} plugin in {timer}ms", pluginContainer.Info.Name, stopwatch.ElapsedMilliseconds); loadContext.Unload(); } @@ -227,6 +199,63 @@ public void ServerReady() public PluginContainer GetPluginContainerByAssembly(Assembly? assembly = null) => this.Plugins.First(x => x.PluginAssembly == (assembly ?? Assembly.GetCallingAssembly())); + private void ConfigureInitialServices(IServerEnvironment env) + { + this.pluginServiceDescriptors.AddLogging((builder) => + { + builder.ClearProviders(); + builder.AddProvider(new LoggerProvider(env.Configuration.LogLevel)); + builder.SetMinimumLevel(env.Configuration.LogLevel); + }); + this.pluginServiceDescriptors.AddSingleton(x => env.Configuration); + } + + private PluginContainer HandlePlugin(PluginContainer pluginContainer) + { + //The plugin still hasn't fully loaded. Probably due to it having a hard dependency + if (pluginContainer.Plugin is null) + return pluginContainer; + + //Inject first wave of services (services initialized by obsidian e.x IServerConfiguration) + PluginServiceHandler.InjectServices(this.serverProvider, pluginContainer, this.logger); + + if (pluginContainer.IsReady) + { + lock (plugins) + { + plugins.Add(pluginContainer); + } + + pluginContainer.Plugin.ConfigureServices(this.pluginServiceDescriptors); + pluginContainer.Plugin.ConfigureRegistry(this.pluginRegistry); + + pluginContainer.Loaded = true; + + InvokeOnLoad(pluginContainer); + } + else + { + lock (stagedPlugins) + { + stagedPlugins.Add(pluginContainer); + } + + if (logger != null) + { + var stageMessage = new System.Text.StringBuilder(50); + stageMessage.Append($"Plugin {pluginContainer.Info.Name} staged"); + if (!pluginContainer.HasDependencies) + stageMessage.Append(", missing dependencies"); + + logger.LogWarning("{}", stageMessage.ToString()); + } + } + + logger?.LogInformation("Loading finished!"); + + return pluginContainer; + } + private void OnPluginSourceRenamed(string oldSource, string newSource) { var renamedPlugin = plugins.FirstOrDefault(plugin => plugin.Source == oldSource) ?? stagedPlugins.FirstOrDefault(plugin => plugin.Source == oldSource); @@ -249,17 +278,7 @@ private void InvokeOnLoad(PluginContainer plugin) if (task.Status == TaskStatus.Faulted) logger?.LogError(task.Exception?.InnerException, "Invoking {pluginName}.OnLoadedAsync faulted.", plugin.Info.Name); - - } - private void InvokeOnUnloading(PluginContainer plugin) - { - var task = plugin.Plugin.OnUnloadingAsync().AsTask(); - if (task.Status == TaskStatus.Created) - task.RunSynchronously(); - - if (task.Status == TaskStatus.Faulted) - logger?.LogError(task.Exception?.InnerException, "Invoking {pluginName}.OnUnloadingAsync faulted.", plugin.Info.Name); } private void InvokeOnServerReady(PluginContainer plugin) @@ -267,7 +286,7 @@ private void InvokeOnServerReady(PluginContainer plugin) var task = plugin.Plugin.OnServerReadyAsync(this.server).AsTask(); if (task.Status == TaskStatus.Created) task.RunSynchronously(); - + if (task.Status == TaskStatus.Faulted) logger?.LogError(task.Exception?.InnerException, "Invoking {pluginName}.OnServerReadyAsync faulted.", plugin.Info.Name); } diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index b7d32708e..ed6af2f7f 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -1,15 +1,12 @@ using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; using Org.BouncyCastle.Crypto; -using System.Buffers; using System.Collections.Frozen; using System.IO; using System.IO.Compression; using System.Reflection; -using System.Runtime.Loader; using System.Security.Cryptography; using System.Text; -using System.Text.Json; namespace Obsidian.Plugins.PluginProviders; public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger logger) @@ -19,7 +16,7 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo public async Task GetPluginAsync(string path) { - await using var fs = new FileStream(path, FileMode.Open); + await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); using var reader = new BinaryReader(fs); var header = Encoding.ASCII.GetString(reader.ReadBytes(4)); @@ -50,6 +47,73 @@ public async Task GetPluginAsync(string path) var loadContext = new PluginLoadContext(pluginAssembly); + var entries = await this.InitializeEntriesAsync(reader, fs); + + var partialContainer = BuildPartialContainer(loadContext, path, entries); + + //Can't load until those plugins are loaded + if (partialContainer.Info.Dependencies.Any(x => x.Priority == DependencyPriority.Hard)) + return partialContainer; + + var mainAssembly = this.InitializePlugin(partialContainer); + + return HandlePlugin(partialContainer, mainAssembly); + } + + internal Assembly InitializePlugin(PluginContainer pluginContainer) + { + var pluginAssembly = pluginContainer.LoadContext.Name; + + var libsWithSymbols = this.ProcessEntries(pluginContainer); + foreach (var lib in libsWithSymbols) + { + var mainLib = pluginContainer.GetFileData($"{lib}.dll"); + var libSymbols = pluginContainer.GetFileData($"{lib}.pdb"); + + pluginContainer.LoadContext.LoadAssembly(mainLib, libSymbols); + } + + var mainPluginEntry = pluginContainer.GetFileData($"{pluginAssembly}.dll"); + var mainPluginPbdEntry = pluginContainer.GetFileData($"{pluginAssembly}.pdb"); + + var mainAssembly = pluginContainer.LoadContext.LoadAssembly(mainPluginEntry!, mainPluginPbdEntry!) + ?? throw new InvalidOperationException("Failed to find main assembly"); + + pluginContainer.PluginAssembly = mainAssembly; + + return mainAssembly; + } + + internal PluginContainer HandlePlugin(PluginContainer pluginContainer, Assembly assembly) + { + Type? pluginType = assembly.GetTypes().FirstOrDefault(type => type.IsSubclassOf(typeof(PluginBase))); + + PluginBase? plugin; + if (pluginType == null || pluginType.GetConstructor([]) == null) + { + plugin = default; + logger.LogError("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); + + throw new InvalidOperationException("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); + } + + logger.LogDebug("Creating plugin instance..."); + plugin = (PluginBase)Activator.CreateInstance(pluginType)!; + + pluginContainer.PluginAssembly = assembly; + pluginContainer.Plugin = plugin; + + pluginContainer.Initialize(); + + return pluginContainer; + } + + /// + /// Steps through the plugin file stream and initializes each file entry found. + /// + /// A dictionary that contains file entries with the key as the FileName and value as . + private async Task> InitializeEntriesAsync(BinaryReader reader, FileStream fs) + { var entryCount = reader.ReadInt32(); var entries = new Dictionary(entryCount); @@ -71,38 +135,49 @@ public async Task GetPluginAsync(string path) var startPos = (int)fs.Position; foreach (var (_, entry) in entries) + { entry.Offset += startPos; - var libsWithSymbols = new List(); - foreach (var (_, entry) in entries) - { - byte[] actualBytes; - if (entry.Length != entry.CompressedLength) - { - var mem = new byte[entry.CompressedLength]; + var data = new byte[entry.CompressedLength]; - var compressedBytesRead = await fs.ReadAsync(mem); + var bytesRead = await fs.ReadAsync(data); - if (compressedBytesRead != entry.CompressedLength) - throw new DataLengthException(); + if (bytesRead != entry.CompressedLength) + throw new DataLengthException(); - await using var ms = new MemoryStream(mem); - await using var ds = new DeflateStream(ms, CompressionMode.Decompress); + entry.rawData = data; + } - await using var deflatedData = new MemoryStream(); + return entries; + } - await ds.CopyToAsync(deflatedData); + private PluginContainer BuildPartialContainer(PluginLoadContext loadContext, string path, + Dictionary entries) + { + var pluginContainer = new PluginContainer + { + LoadContext = loadContext, + Source = path, + FileEntries = entries.ToFrozenDictionary(), + }; - if (deflatedData.Length != entry.Length) - throw new DataLengthException(); + pluginContainer.Initialize(); + + return pluginContainer; + } - actualBytes = deflatedData.ToArray(); - } - else - { - actualBytes = new byte[entry.Length]; - await fs.ReadAsync(actualBytes); - } + + /// + /// Goes and loads any assemblies found into the . + /// + private List ProcessEntries(PluginContainer pluginContainer) + { + var pluginAssembly = pluginContainer.LoadContext.Name; + + var libsWithSymbols = new List(); + foreach (var (_, entry) in pluginContainer.FileEntries) + { + var actualBytes = entry.GetData(); var name = Path.GetFileNameWithoutExtension(entry.Name); //Don't load this assembly wait @@ -112,66 +187,17 @@ public async Task GetPluginAsync(string path) //TODO LOAD OTHER FILES SOMEWHERE if (entry.Name.EndsWith(".dll")) { - if(entries.ContainsKey(entry.Name.Replace(".dll", ".pdb"))) + if (pluginContainer.FileEntries.ContainsKey(entry.Name.Replace(".dll", ".pdb"))) { //Library has debug symbols load in last libsWithSymbols.Add(entry.Name.Replace(".dll", ".pdb")); continue; } - loadContext.LoadAssembly(actualBytes); + pluginContainer.LoadContext.LoadAssembly(actualBytes); } } - foreach(var lib in libsWithSymbols) - { - var mainLib = await entries[$"{lib}.dll"].GetDataAsync(fs); - var libSymbols = await entries[$"{lib}.pdb"].GetDataAsync(fs); - - loadContext.LoadAssembly(mainLib, libSymbols); - } - - var mainPluginEntry = entries[$"{pluginAssembly}.dll"]; - var mainPluginPbdEntry = entries[$"{pluginAssembly}.pdb"]; - - var mainAssembly = loadContext.LoadAssembly(await mainPluginEntry.GetDataAsync(fs), await mainPluginPbdEntry.GetDataAsync(fs)); - - if (mainAssembly is null) - throw new InvalidOperationException("Failed to find main assembly"); - - await fs.DisposeAsync(); - - return await HandlePluginAsync(loadContext, mainAssembly, path, entries); - } - - internal async Task HandlePluginAsync(PluginLoadContext loadContext, Assembly assembly, - string path, Dictionary entries) - { - Type? pluginType = assembly.GetTypes().FirstOrDefault(type => type.IsSubclassOf(typeof(PluginBase))); - - PluginBase? plugin; - if (pluginType == null || pluginType.GetConstructor([]) == null) - { - plugin = default; - logger.LogError("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); - - throw new InvalidOperationException("Loaded assembly contains no type implementing PluginBase with public parameterless constructor."); - } - - logger.LogInformation("Creating plugin instance..."); - plugin = (PluginBase)Activator.CreateInstance(pluginType)!; - - var pluginContainer = new PluginContainer - { - Plugin = plugin, - FileEntries = entries.ToFrozenDictionary(), - LoadContext = loadContext, - PluginAssembly = assembly, - Source = path - }; - - await pluginContainer.InitializeAsync(); - - return pluginContainer; + return libsWithSymbols; } } diff --git a/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs b/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs index e52d79bb9..928e24f6c 100644 --- a/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs +++ b/Obsidian/Plugins/PluginProviders/PluginLoadContext.cs @@ -4,7 +4,7 @@ namespace Obsidian.Plugins.PluginProviders; -internal sealed class PluginLoadContext(string name) : AssemblyLoadContext(name: name, isCollectible: true) +public sealed class PluginLoadContext(string name) : AssemblyLoadContext(name: name, isCollectible: true) { public List Dependencies { get; } = []; @@ -14,7 +14,7 @@ internal sealed class PluginLoadContext(string name) : AssemblyLoadContext(name: using var pbdStream = pbdBytes != null ? new MemoryStream(pbdBytes, false) : null; var asm = this.LoadFromStream(mainStream, pbdStream); - + return asm; } diff --git a/Obsidian/Server.cs b/Obsidian/Server.cs index a51517403..e43d812ab 100644 --- a/Obsidian/Server.cs +++ b/Obsidian/Server.cs @@ -256,10 +256,9 @@ public async Task RunAsync() Directory.CreateDirectory("plugins"); - PluginManager.DirectoryWatcher.Filters = [".obby"]; - PluginManager.DirectoryWatcher.Watch("plugins"); + await PluginManager.LoadPluginsAsync(); - await Task.WhenAll(Configuration.DownloadPlugins.Select(path => PluginManager.LoadPluginAsync(path))); + //await Task.WhenAll(Configuration.DownloadPlugins.Select(path => PluginManager.LoadPluginAsync(path))); if (!Configuration.OnlineMode) _logger.LogInformation("Starting in offline mode..."); diff --git a/Obsidian/Utilities/Extensions.cs b/Obsidian/Utilities/Extensions.cs index e7186a09c..d55e429cd 100644 --- a/Obsidian/Utilities/Extensions.cs +++ b/Obsidian/Utilities/Extensions.cs @@ -109,7 +109,7 @@ public static bool IsConnected(this ConnectionContext context) { return !context.ConnectionClosed.IsCancellationRequested; } - + // Derived from https://gist.github.com/ammaraskar/7b4a3f73bee9dc4136539644a0f27e63 [SuppressMessage("Roslyn", "CA5350", Justification = "SHA1 is required by the Minecraft protocol.")] public static string MinecraftShaDigest(this IEnumerable data) @@ -134,13 +134,14 @@ public static string MinecraftShaDigest(this IEnumerable data) } public static string ToJson(this object? value, JsonSerializerOptions? options = null) => JsonSerializer.Serialize(value, options ?? Globals.JsonOptions); - public static T? FromJson(this string value) => JsonSerializer.Deserialize(value, Globals.JsonOptions); + public static T? FromJson(this string value, JsonSerializerOptions? options = null) => JsonSerializer.Deserialize(value, options ?? Globals.JsonOptions); + public static TValue? FromJson(this byte[] jsonData, JsonSerializerOptions? options = null) => + JsonSerializer.Deserialize(jsonData, options ?? Globals.JsonOptions); public static ValueTask FromJsonAsync(this Stream stream, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) => JsonSerializer.DeserializeAsync(stream, options ?? Globals.JsonOptions, cancellationToken); public static Task ToJsonAsync(this object? value, Stream stream, CancellationToken cancellationToken = default) => JsonSerializer.SerializeAsync(stream, value, Globals.JsonOptions, cancellationToken); - public static bool Contains(this ReadOnlySpan span, T value) where T : unmanaged { for (int i = 0; i < span.Length; i++) diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 998254263..cd4881fdc 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -36,7 +36,8 @@ - + From e376c7b8655903db7c3be2844585957c0a0d627a Mon Sep 17 00:00:00 2001 From: Tides Date: Sat, 2 Mar 2024 21:45:43 -0500 Subject: [PATCH 09/21] Load plugin after dependents are loaded --- Obsidian.API/Plugins/IPluginInfo.cs | 10 ++-------- Obsidian/Plugins/PluginContainer.cs | 4 ++-- Obsidian/Plugins/PluginManager.cs | 15 +++++++++------ .../PluginProviders/PackedPluginProvider.cs | 17 +++++++++++------ Obsidian/Server.cs | 2 +- SamplePlugin/plugin.json | 9 ++++++++- 6 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Obsidian.API/Plugins/IPluginInfo.cs b/Obsidian.API/Plugins/IPluginInfo.cs index 91ac57217..4df0f666d 100644 --- a/Obsidian.API/Plugins/IPluginInfo.cs +++ b/Obsidian.API/Plugins/IPluginInfo.cs @@ -13,15 +13,9 @@ public interface IPluginInfo public readonly struct PluginDependency { - public required string Name { get; init; } + public required string Id { get; init; } public required string Version { get; init; } - public DependencyPriority Priority { get; init; } -} - -public enum DependencyPriority -{ - Soft, - Hard + public bool Required { get; init; } } diff --git a/Obsidian/Plugins/PluginContainer.cs b/Obsidian/Plugins/PluginContainer.cs index 6b22bcf37..af136d8c0 100644 --- a/Obsidian/Plugins/PluginContainer.cs +++ b/Obsidian/Plugins/PluginContainer.cs @@ -68,8 +68,8 @@ internal void Initialize() } //TODO PLUGINS SHOULD USE VERSION CLASS TO SPECIFY VERSION - public bool IsDependency(string pluginName) => - this.Info.Dependencies.Any(x => x.Name == pluginName); + public bool IsDependency(string pluginId) => + this.Info.Dependencies.Any(x => x.Id == pluginId); /// /// Inject the scoped services into diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index d1d9e67b8..a439ce409 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -11,6 +11,7 @@ using System.Collections.Immutable; using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection; namespace Obsidian.Plugins; @@ -92,16 +93,18 @@ public async Task LoadPluginsAsync() { var pluginContainer = await this.LoadPluginAsync(file); + foreach (var canLoad in waitingForDepend.Where(x => x.IsDependency(pluginContainer.Info.Id)).ToList()) + { + packedPluginProvider.InitializePlugin(canLoad); + this.HandlePlugin(canLoad); + + waitingForDepend.Remove(canLoad); + } + if (pluginContainer.Plugin is null) waitingForDepend.Add(pluginContainer); } - foreach (var pluginToLoad in waitingForDepend) - { - packedPluginProvider.InitializePlugin(pluginToLoad); - this.HandlePlugin(pluginToLoad); - } - DirectoryWatcher.Watch("plugins"); } diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index ed6af2f7f..86b5e1811 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -52,8 +52,13 @@ public async Task GetPluginAsync(string path) var partialContainer = BuildPartialContainer(loadContext, path, entries); //Can't load until those plugins are loaded - if (partialContainer.Info.Dependencies.Any(x => x.Priority == DependencyPriority.Hard)) + if (partialContainer.Info.Dependencies.Any(x => x.Required)) + { + var str = partialContainer.Info.Dependencies.Length > 1 ? "has multiple hard dependencies." : + $"has a hard dependecy on {partialContainer.Info.Dependencies.First().Id}."; + this.logger.LogWarning("{name} {message}. Will Attempt to load after.", partialContainer.Info.Name, str); return partialContainer; + } var mainAssembly = this.InitializePlugin(partialContainer); @@ -67,16 +72,16 @@ internal Assembly InitializePlugin(PluginContainer pluginContainer) var libsWithSymbols = this.ProcessEntries(pluginContainer); foreach (var lib in libsWithSymbols) { - var mainLib = pluginContainer.GetFileData($"{lib}.dll"); - var libSymbols = pluginContainer.GetFileData($"{lib}.pdb"); + var mainLib = pluginContainer.GetFileData($"{lib}.dll")!; + var libSymbols = pluginContainer.GetFileData($"{lib}.pdb")!; pluginContainer.LoadContext.LoadAssembly(mainLib, libSymbols); } - var mainPluginEntry = pluginContainer.GetFileData($"{pluginAssembly}.dll"); - var mainPluginPbdEntry = pluginContainer.GetFileData($"{pluginAssembly}.pdb"); + var mainPluginEntry = pluginContainer.GetFileData($"{pluginAssembly}.dll")!; + var mainPluginPbdEntry = pluginContainer.GetFileData($"{pluginAssembly}.pdb")!; - var mainAssembly = pluginContainer.LoadContext.LoadAssembly(mainPluginEntry!, mainPluginPbdEntry!) + var mainAssembly = pluginContainer.LoadContext.LoadAssembly(mainPluginEntry, mainPluginPbdEntry!) ?? throw new InvalidOperationException("Failed to find main assembly"); pluginContainer.PluginAssembly = mainAssembly; diff --git a/Obsidian/Server.cs b/Obsidian/Server.cs index e43d812ab..c9ba08831 100644 --- a/Obsidian/Server.cs +++ b/Obsidian/Server.cs @@ -125,7 +125,7 @@ public Server( CommandsHandler = commandHandler; - PluginManager = new PluginManager(this.serviceProvider, this, eventDispatcher, CommandsHandler, _logger); + PluginManager = new PluginManager(this.serviceProvider, this, eventDispatcher, CommandsHandler, loggerFactory.CreateLogger()); _logger.LogDebug("Registering commands..."); CommandsHandler.RegisterCommandClass(null); diff --git a/SamplePlugin/plugin.json b/SamplePlugin/plugin.json index 9602ed78c..8ff5b75dd 100644 --- a/SamplePlugin/plugin.json +++ b/SamplePlugin/plugin.json @@ -6,5 +6,12 @@ "Obsidian Team" ], "description": "My sample plugin.", - "projectUrl": "https://github.com/ObsidianMC/Obsidian" + "projectUrl": "https://github.com/ObsidianMC/Obsidian", + "dependencies": [ + { + "id": "test_plugin", + "version": "1.0.0-beta", + "required": true + } + ] } \ No newline at end of file From 7f30aa7cfe436fe99ee58c9c70850e5eb9073a62 Mon Sep 17 00:00:00 2001 From: Tides Date: Mon, 4 Mar 2024 23:01:50 -0500 Subject: [PATCH 10/21] Check if signature is valid --- Obsidian/Obsidian.csproj | 4 +- Obsidian/Plugins/PluginContainer.cs | 2 + Obsidian/Plugins/PluginManager.cs | 40 +++++++++++++++++-- .../PluginProviders/PackedPluginProvider.cs | 33 +++++++++++---- SamplePlugin/SamplePlugin.csproj | 9 +++-- SamplePlugin/signing_key.xml | 16 ++++++++ 6 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 SamplePlugin/signing_key.xml diff --git a/Obsidian/Obsidian.csproj b/Obsidian/Obsidian.csproj index d9f707bad..aca9ae955 100644 --- a/Obsidian/Obsidian.csproj +++ b/Obsidian/Obsidian.csproj @@ -53,8 +53,8 @@ - - + + diff --git a/Obsidian/Plugins/PluginContainer.cs b/Obsidian/Plugins/PluginContainer.cs index af136d8c0..94d50fdc1 100644 --- a/Obsidian/Plugins/PluginContainer.cs +++ b/Obsidian/Plugins/PluginContainer.cs @@ -30,6 +30,8 @@ public sealed class PluginContainer : IDisposable public FrozenDictionary FileEntries { get; internal set; } = default!; public required string Source { get; set; } + public required bool ValidSignature { get; init; } + public bool HasDependencies { get; private set; } = true; public bool IsReady => HasDependencies; public bool Loaded { get; internal set; } diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index a439ce409..3aa5265ae 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -8,11 +8,14 @@ using Obsidian.Plugins.ServiceProviders; using Obsidian.Registries; using Obsidian.Services; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Crypto.Utilities; +using Org.BouncyCastle.Security; using System.Collections.Immutable; using System.Diagnostics; using System.IO; -using System.Linq; using System.Reflection; +using System.Security.Cryptography; namespace Obsidian.Plugins; @@ -24,7 +27,7 @@ public sealed class PluginManager private readonly List plugins = []; private readonly List stagedPlugins = []; - private readonly List acceptedKeys = []; + private readonly List acceptedKeys = []; private readonly IServiceProvider serverProvider; private readonly IServer server; @@ -32,7 +35,7 @@ public sealed class PluginManager private readonly IPluginRegistry pluginRegistry; private readonly IServiceCollection pluginServiceDescriptors = new ServiceCollection(); - public ImmutableArray AcceptedKeys => acceptedKeys.ToImmutableArray(); + public ImmutableArray AcceptedKeys => acceptedKeys.ToImmutableArray(); /// /// List of all loaded plugins. @@ -82,10 +85,38 @@ public PluginManager(IServiceProvider serverProvider, IServer server, DirectoryWatcher.FileRenamed += OnPluginSourceRenamed; DirectoryWatcher.FileDeleted += OnPluginSourceDeleted; } - public async Task LoadPluginsAsync() { + //TODO talk about what format we should support + await using var acceptedKeysFileStream = new FileStream("accepted_keys", FileMode.OpenOrCreate); + + if(acceptedKeysFileStream.Length > 0) + { + using var sr = new StreamReader(acceptedKeysFileStream); + var line = ""; + while((line = await sr.ReadLineAsync()) != null) + { + //ssh-rsa AAAAB3.... + var key = line.Split()[1];//Try to get the base 64 encoded only RSA keys are supported. + var keyParams = OpenSshPublicKeyUtilities.ParsePublicKey(Convert.FromBase64String(key)); + + try + { + var rsaKeyParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)keyParams); + + acceptedKeys.Add(rsaKeyParams); + } + catch(Exception ex) + { + this.logger.LogWarning(ex, "Failed to parse public key."); + } + + + this.logger.LogDebug("Added key {key}", line); + } + } + var files = Directory.GetFiles("plugins", "*.obby", SearchOption.AllDirectories); var waitingForDepend = new List(); @@ -100,6 +131,7 @@ public async Task LoadPluginsAsync() waitingForDepend.Remove(canLoad); } + if (pluginContainer.Plugin is null) waitingForDepend.Add(pluginContainer); diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index 86b5e1811..073dba1b5 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -1,9 +1,10 @@ using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; using Org.BouncyCastle.Crypto; +using Org.BouncyCastle.Crypto.Utilities; +using Org.BouncyCastle.Security; using System.Collections.Frozen; using System.IO; -using System.IO.Compression; using System.Reflection; using System.Security.Cryptography; using System.Text; @@ -11,6 +12,9 @@ namespace Obsidian.Plugins.PluginProviders; public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger logger) { + private const int SignatureLength = 384; + private const int HashLength = 20; + private readonly PluginManager pluginManager = pluginManager; private readonly ILogger logger = logger; @@ -26,12 +30,11 @@ public async Task GetPluginAsync(string path) //TODO save api version somewhere var apiVersion = reader.ReadString(); - var hash = reader.ReadBytes(20); - var signature = reader.ReadBytes(256); + var hash = reader.ReadBytes(HashLength); + var signature = reader.ReadBytes(SignatureLength); var dataLength = reader.ReadInt32(); var curPos = fs.Position; - using (var sha1 = SHA1.Create()) { var verifyHash = await sha1.ComputeHashAsync(fs); @@ -40,6 +43,19 @@ public async Task GetPluginAsync(string path) throw new InvalidDataException("File integrity does not match specified hash."); } + var f = new RSAPKCS1SignatureDeformatter(); + f.SetHashAlgorithm("SHA1"); + + using var v = RSA.Create(); + var isSigValid = false; + foreach (var key in this.pluginManager.AcceptedKeys) + { + v.ImportParameters(key); + f.SetKey(v); + + isSigValid = f.VerifySignature(hash, signature); + } + fs.Position = curPos; var pluginAssembly = reader.ReadString(); @@ -49,13 +65,13 @@ public async Task GetPluginAsync(string path) var entries = await this.InitializeEntriesAsync(reader, fs); - var partialContainer = BuildPartialContainer(loadContext, path, entries); + var partialContainer = BuildPartialContainer(loadContext, path, entries, isSigValid); //Can't load until those plugins are loaded - if (partialContainer.Info.Dependencies.Any(x => x.Required)) + if (partialContainer.Info.Dependencies.Any(x => x.Required && !this.pluginManager.Plugins.Any(d => d.Info.Id == x.Id))) { var str = partialContainer.Info.Dependencies.Length > 1 ? "has multiple hard dependencies." : - $"has a hard dependecy on {partialContainer.Info.Dependencies.First().Id}."; + $"has a hard dependency on {partialContainer.Info.Dependencies.First().Id}."; this.logger.LogWarning("{name} {message}. Will Attempt to load after.", partialContainer.Info.Name, str); return partialContainer; } @@ -157,13 +173,14 @@ private async Task> InitializeEntriesAsync(B } private PluginContainer BuildPartialContainer(PluginLoadContext loadContext, string path, - Dictionary entries) + Dictionary entries, bool validSignature) { var pluginContainer = new PluginContainer { LoadContext = loadContext, Source = path, FileEntries = entries.ToFrozenDictionary(), + ValidSignature = validSignature }; pluginContainer.Initialize(); diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index cd4881fdc..f72d43113 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -13,6 +13,7 @@ 1.0.0.0 1.0.0 bin/$(Configuration)/ + signing_key.xml @@ -20,7 +21,7 @@ runtime - + @@ -31,9 +32,9 @@ - - PreserveNewest - + + PreserveNewest + diff --git a/SamplePlugin/signing_key.xml b/SamplePlugin/signing_key.xml new file mode 100644 index 000000000..7cb09cbb8 --- /dev/null +++ b/SamplePlugin/signing_key.xml @@ -0,0 +1,16 @@ + + + xgXiI3LxZhBvCU0XEqITCSPHhOhu4yMiAheBDgU4EphUoLvsMo6VDwMpoafeM6585n5D/5aC+zbxZCPAI4UD2FknCVikAYqOdpUYAhwIhTVoI1LrCF8/fL+gEH3hz6psu1ddpR2j2qzMZNe5gWfJv4acYLGR0rQ3cro2JOznvGKLVWq6fBBSWh3z5QujolskVGaaWerMF1i5A0MykWgucKrc1m9EnE17g8/z0zdtLZSNLkfVkzzqdNWD2g7jIowtfgw5GVUtYS8/aW9uDUyuDh+9648uEyQZkOe+ZZpq1fGUyhQO/zBDj48atyuG1dLg6qQ+XuoVB715Lgrzn2S/43IsNitvV219XvAFkcXmg+dvDJRaAN9PdJbGLFjUWoY0QNJG2mj2ZTMm9Ge8bdX+E23Zi1OGlmvFGllg+guqJWspPzu8BFsVY2NfZA7XhcCT99KrC3MK8Xmn8RCXpW6hD3KUL7YqYCTdTl7HJ1ozDqOSwa9WRHtzqWP0noBnida/ + + AQAB +

+ 6VTDu0JIlGyj0MEqnpcNggTaQuywn8KPAmLQYtfPQM4tDH6fJS/2Zz9WUnWAwdg5rJ1vPDHHfHExl2XeUJm/ORYiVYxWCbUmgUbRxsFxvyuGceFaGSCetMiJAUxgaNYVGBwPROPJs5G0PkdyjTMU9L3TQK+HkZM37ZcHkhtkOAVz/ybvUjd35Q/pIqxiwfeXj6h95VoY0C9dproh3NNsaZeigjVXAL19nAp4AeBT+z8/8qRzHuFKCA+986wIGpuz +

+ + 2UL2K6tTw3niPUINE1RhoMJ+pzenrvTdv2DZUeqYXju9xDsDP2tggsR04J0bNQmtie5pVAsuKQGyKJS9Bq2ytFIwpA33JS1/GO7wgAUuUyTJqtcrcy8SsX4T67HsrFX8JuvhUC3dwbFcewH0Y5FJ39LsdaFoNWaW379XlwuNCJM861MYmjDqRoPWD+rxXIi5UShfzy+15jiBj6svj4Rx9FchpWA0wp4saWgPG97EB+RtVVx+yj7I9g+zHa0d9OLF + + kfnG7pt8XudM4WhIKqmj+MjiZ+Y+ZeTJpZt+dahAcHHMuzlohVJpXJTCg5ohsKR/CKACHG2kORV0ChzgJhraCHPxjQXcOfyMF2rgCPGzIP+xAAQVYw0GXWHsXNSUqGHvFoNPhjDgWqh4VFJMt0vS+37a8Gggb5Hj9o97XAwo+ednd6S7Kvuq1bLLjza8Hk+xEV4/TUK9rVlExO9ECix3ceQ89is6wCrMr9fY1ouq3s/mSP5bS2F0+HAozY6+Dy+D + uGLAEafB2zBh9aNM/Y6rnIf0fW4afb2LRJSpW4BB+BezogqDYxt6OQEKQGaVZJnmEh2ofo9OibbKwO4azsQz2h228kR377GBIGQI8F11R1L9ZGRgl8znjIN33JyQRGJlMZMXEbkbbvtbhXnPM4FmtPKJg/uMOWXrvdDiQWqoMxLirMVFJ/dDnLzeu3ulg+b1gA9H4MqZEd369zdNkfvRypD4GSZhIJqlDoAl8n1I2Xnf4IYzhOlsHIiOdy0pM2AR + IJqfoKt3YSS43Jz5sGW3Mct9V1156I2YTNFvVSJIxTFdPtlBc6I+qWmq2EUyZzvJNY8ppVCuXprzVOhol1wQQriPV6w0fXvdQ4X9K2k6m95v/Ohstp0dT08MpWPKPquGzYIb6mS4k8HVOEgsmNU9ImekxhIfmK/gw66/JUF4VAF7wKXmHH9Vi3wXc4xwpsZAfbvTM5+C/b2vhHR/0KbWhTWHtVgygoCfc2RocWkLW2nUosE9T6whzCymf/60ZVGW + LnAoxRlqhP3rnCnI9GSvxB/w5TlA7+FIU2dBq5ELwkiY6AzJ6l1HlXqvY6qjUElmHwOTXLfmRZVv7IT5xRqneTAHGIkCBR6CJEr4k14RnXOV1VrXpWH42H9zwQpUT0fvMcveOR+HUnvdfvScGz3EsTaXK5HY2anLqwsCMYyhF6ugc0qRKxEEzv9hN+CP4j4ved+J6PZIIF19Hce/bxwYb/GR0nPLikAdbB7zViAUaXEB3s2Xx2ysqvN7dUKyf2KRyucaAS8KczvrxAXPhYR7peNX5IoyUVsO95DFjWdK8bBISyKqf2HXUUcnQZo6Ukf+tv5Vua8dwgYtnt8UZO0CYGWU3UBH22wcB5WKvGa7FV5UMfnCjAc2ELM5yRKo0FnABeZ2btJ3v4r1G64UbD0atDpTmYF+7m0STqMa5OsOdRLqhy3Z1xficX8uWQtm9Gq8kQOqLOyLZGraQPRzhmXRCKIZwh4sNCZ28W394oLOR9cZcHy2Kjp2pBNwKkT/dPqJ + \ No newline at end of file From 9c85b1f6289b7fa32985fe141df7eca2a1fa7ed4 Mon Sep 17 00:00:00 2001 From: Tides Date: Mon, 4 Mar 2024 23:22:42 -0500 Subject: [PATCH 11/21] Add AllowUntrustedPlugins to config and check against that --- .../_Interfaces/IServerConfiguration.cs | 5 ++ Obsidian/Plugins/PluginManager.cs | 14 ++-- .../PluginProviders/PackedPluginProvider.cs | 64 +++++++++++++------ Obsidian/Utilities/ServerConfiguration.cs | 2 + 4 files changed, 60 insertions(+), 25 deletions(-) diff --git a/Obsidian.API/_Interfaces/IServerConfiguration.cs b/Obsidian.API/_Interfaces/IServerConfiguration.cs index f2f6b5078..2b279cbed 100644 --- a/Obsidian.API/_Interfaces/IServerConfiguration.cs +++ b/Obsidian.API/_Interfaces/IServerConfiguration.cs @@ -11,6 +11,11 @@ public interface IServerConfiguration /// public bool CanThrottle => this.ConnectionThrottle > 0; + /// + /// Determines where or not the server should load plugins that don't have a valid signature. + /// + public bool AllowUntrustedPlugins { get; set; } + /// /// Allows the server to advertise itself as a LAN server to devices on your network. /// diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 3aa5265ae..4b411e49d 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -22,6 +22,7 @@ namespace Obsidian.Plugins; public sealed class PluginManager { internal readonly ILogger logger; + internal readonly IServer server; private static PackedPluginProvider packedPluginProvider = default!; @@ -30,7 +31,6 @@ public sealed class PluginManager private readonly List acceptedKeys = []; private readonly IServiceProvider serverProvider; - private readonly IServer server; private readonly CommandHandler commandHandler; private readonly IPluginRegistry pluginRegistry; private readonly IServiceCollection pluginServiceDescriptors = new ServiceCollection(); @@ -98,7 +98,8 @@ public async Task LoadPluginsAsync() while((line = await sr.ReadLineAsync()) != null) { //ssh-rsa AAAAB3.... - var key = line.Split()[1];//Try to get the base 64 encoded only RSA keys are supported. + //Try to get the base 64 encoded section. Only RSA keys are supported. + var key = line.Split()[1]; var keyParams = OpenSshPublicKeyUtilities.ParsePublicKey(Convert.FromBase64String(key)); try @@ -124,6 +125,9 @@ public async Task LoadPluginsAsync() { var pluginContainer = await this.LoadPluginAsync(file); + if (pluginContainer is null) + continue; + foreach (var canLoad in waitingForDepend.Where(x => x.IsDependency(pluginContainer.Info.Id)).ToList()) { packedPluginProvider.InitializePlugin(canLoad); @@ -145,13 +149,13 @@ public async Task LoadPluginsAsync() /// /// Path to load the plugin from. Can point either to local OBBY or DLL. /// Loaded plugin. If loading failed, property will be null. - public async Task LoadPluginAsync(string path) + public async Task LoadPluginAsync(string path) { try { - PluginContainer plugin = await packedPluginProvider.GetPluginAsync(path).ConfigureAwait(false); + var plugin = await packedPluginProvider.GetPluginAsync(path).ConfigureAwait(false); - return HandlePlugin(plugin); + return plugin is null ? null : HandlePlugin(plugin); } catch (Exception ex) { diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index 073dba1b5..62921ebf6 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -2,6 +2,7 @@ using Obsidian.API.Plugins; using Org.BouncyCastle.Crypto; using Org.BouncyCastle.Crypto.Utilities; +using Org.BouncyCastle.Pqc.Crypto.SphincsPlus; using Org.BouncyCastle.Security; using System.Collections.Frozen; using System.IO; @@ -18,7 +19,9 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo private readonly PluginManager pluginManager = pluginManager; private readonly ILogger logger = logger; - public async Task GetPluginAsync(string path) + + + public async Task GetPluginAsync(string path) { await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); using var reader = new BinaryReader(fs); @@ -35,27 +38,12 @@ public async Task GetPluginAsync(string path) var dataLength = reader.ReadInt32(); var curPos = fs.Position; - using (var sha1 = SHA1.Create()) - { - var verifyHash = await sha1.ComputeHashAsync(fs); - - if (!verifyHash.SequenceEqual(hash)) - throw new InvalidDataException("File integrity does not match specified hash."); - } - - var f = new RSAPKCS1SignatureDeformatter(); - f.SetHashAlgorithm("SHA1"); - using var v = RSA.Create(); - var isSigValid = false; - foreach (var key in this.pluginManager.AcceptedKeys) - { - v.ImportParameters(key); - f.SetKey(v); + //Don't load untrusted plugins + var isSigValid = await this.TryValidatePluginAsync(fs, hash, signature, path); + if (!isSigValid) + return null; - isSigValid = f.VerifySignature(hash, signature); - } - fs.Position = curPos; var pluginAssembly = reader.ReadString(); @@ -129,6 +117,42 @@ internal PluginContainer HandlePlugin(PluginContainer pluginContainer, Assembly return pluginContainer; } + /// + /// Verifies the file hash and tries to validate the signature + /// + /// + private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, byte[] signature, string path) + { + using (var sha1 = SHA1.Create()) + { + var verifyHash = await sha1.ComputeHashAsync(fs); + + if (!verifyHash.SequenceEqual(hash)) + { + this.logger.LogWarning("File {filePath} integrity does not match specified hash.", path); + return false; + } + } + + var deformatter = new RSAPKCS1SignatureDeformatter(); + deformatter.SetHashAlgorithm("SHA1"); + + var isSigValid = true; + if (!this.pluginManager.server.Configuration.AllowUntrustedPlugins) + { + using var rsa = RSA.Create(); + foreach (var key in this.pluginManager.AcceptedKeys) + { + rsa.ImportParameters(key); + deformatter.SetKey(rsa); + + isSigValid = deformatter.VerifySignature(hash, signature); + } + } + + return isSigValid; + } + /// /// Steps through the plugin file stream and initializes each file entry found. /// diff --git a/Obsidian/Utilities/ServerConfiguration.cs b/Obsidian/Utilities/ServerConfiguration.cs index 965dd5837..5d9ad9d10 100644 --- a/Obsidian/Utilities/ServerConfiguration.cs +++ b/Obsidian/Utilities/ServerConfiguration.cs @@ -19,6 +19,8 @@ public sealed class ServerConfiguration : IServerConfiguration public bool AllowOperatorRequests { get; set; } = true; + public bool AllowUntrustedPlugins { get; set; } = true; + /// /// If true, each login/client gets a random username where multiple connections from the same host will be allowed. /// From 47cad05b01a17c5bb95ee94a687c34c4e1429f05 Mon Sep 17 00:00:00 2001 From: Tides Date: Tue, 5 Mar 2024 17:03:27 -0500 Subject: [PATCH 12/21] Use SHA384 --- .../accepted_keys/obsidian.pub.xml | 4 ++ Obsidian/Plugins/PluginManager.cs | 41 +++++-------------- .../PluginProviders/PackedPluginProvider.cs | 15 +++---- 3 files changed, 22 insertions(+), 38 deletions(-) create mode 100644 Obsidian.ConsoleApp/accepted_keys/obsidian.pub.xml diff --git a/Obsidian.ConsoleApp/accepted_keys/obsidian.pub.xml b/Obsidian.ConsoleApp/accepted_keys/obsidian.pub.xml new file mode 100644 index 000000000..d0cf80a0b --- /dev/null +++ b/Obsidian.ConsoleApp/accepted_keys/obsidian.pub.xml @@ -0,0 +1,4 @@ + + AMYF4iNy8WYQbwlNFxKiEwkjx4TobuMjIgIXgQ4FOBKYVKC77DKOlQ8DKaGn3jOufOZ+Q/+Wgvs28WQjwCOFA9hZJwlYpAGKjnaVGAIcCIU1aCNS6whfP3y/oBB94c+qbLtXXaUdo9qszGTXuYFnyb+GnGCxkdK0N3K6NiTs57xii1VqunwQUlod8+ULo6JbJFRmmlnqzBdYuQNDMpFoLnCq3NZvRJxNe4PP89M3bS2UjS5H1ZM86nTVg9oO4yKMLX4MORlVLWEvP2lvbg1Mrg4fveuPLhMkGZDnvmWaatXxlMoUDv8wQ4+PGrcrhtXS4OqkPl7qFQe9eS4K859kv+NyLDYrb1dtfV7wBZHF5oPnbwyUWgDfT3SWxixY1FqGNEDSRtpo9mUzJvRnvG3V/hNt2YtThpZrxRpZYPoLqiVrKT87vARbFWNjX2QO14XAk/fSqwtzCvF5p/EQl6VuoQ9ylC+2KmAk3U5exydaMw6jksGvVkR7c6lj9J6AZ4nWvw== + AQAB + \ No newline at end of file diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 4b411e49d..5d49f1b42 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -8,9 +8,6 @@ using Obsidian.Plugins.ServiceProviders; using Obsidian.Registries; using Obsidian.Services; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Crypto.Utilities; -using Org.BouncyCastle.Security; using System.Collections.Immutable; using System.Diagnostics; using System.IO; @@ -85,42 +82,24 @@ public PluginManager(IServiceProvider serverProvider, IServer server, DirectoryWatcher.FileRenamed += OnPluginSourceRenamed; DirectoryWatcher.FileDeleted += OnPluginSourceDeleted; } - + public async Task LoadPluginsAsync() { //TODO talk about what format we should support - await using var acceptedKeysFileStream = new FileStream("accepted_keys", FileMode.OpenOrCreate); + var acceptedKeyFiles = Directory.GetFiles("accepted_keys"); - if(acceptedKeysFileStream.Length > 0) + using var rsa = RSA.Create(); + foreach (var certFile in acceptedKeyFiles) { - using var sr = new StreamReader(acceptedKeysFileStream); - var line = ""; - while((line = await sr.ReadLineAsync()) != null) - { - //ssh-rsa AAAAB3.... - //Try to get the base 64 encoded section. Only RSA keys are supported. - var key = line.Split()[1]; - var keyParams = OpenSshPublicKeyUtilities.ParsePublicKey(Convert.FromBase64String(key)); - - try - { - var rsaKeyParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)keyParams); - - acceptedKeys.Add(rsaKeyParams); - } - catch(Exception ex) - { - this.logger.LogWarning(ex, "Failed to parse public key."); - } - - - this.logger.LogDebug("Added key {key}", line); - } + var xml = await File.ReadAllTextAsync(certFile); + rsa.FromXmlString(xml); + + this.acceptedKeys.Add(rsa.ExportParameters(false)); } var files = Directory.GetFiles("plugins", "*.obby", SearchOption.AllDirectories); - var waitingForDepend = new List(); + var waitingForDepend = new List(); foreach (var file in files) { var pluginContainer = await this.LoadPluginAsync(file); @@ -135,7 +114,7 @@ public async Task LoadPluginsAsync() waitingForDepend.Remove(canLoad); } - + if (pluginContainer.Plugin is null) waitingForDepend.Add(pluginContainer); diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index 62921ebf6..b2e148a6c 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -19,8 +19,6 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo private readonly PluginManager pluginManager = pluginManager; private readonly ILogger logger = logger; - - public async Task GetPluginAsync(string path) { await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -123,9 +121,9 @@ internal PluginContainer HandlePlugin(PluginContainer pluginContainer, Assembly /// private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, byte[] signature, string path) { - using (var sha1 = SHA1.Create()) + using (var sha384 = SHA384.Create()) { - var verifyHash = await sha1.ComputeHashAsync(fs); + var verifyHash = await sha384.ComputeHashAsync(fs); if (!verifyHash.SequenceEqual(hash)) { @@ -135,18 +133,21 @@ private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, byte } var deformatter = new RSAPKCS1SignatureDeformatter(); - deformatter.SetHashAlgorithm("SHA1"); + deformatter.SetHashAlgorithm("SHA384"); var isSigValid = true; if (!this.pluginManager.server.Configuration.AllowUntrustedPlugins) { using var rsa = RSA.Create(); - foreach (var key in this.pluginManager.AcceptedKeys) + foreach (var rsaParameter in this.pluginManager.AcceptedKeys) { - rsa.ImportParameters(key); + rsa.ImportParameters(rsaParameter); deformatter.SetKey(rsa); isSigValid = deformatter.VerifySignature(hash, signature); + + if (isSigValid) + break; } } From ff18da20982b399ab936b4c77b7d4987719e9c86 Mon Sep 17 00:00:00 2001 From: Tides Date: Tue, 5 Mar 2024 18:53:02 -0500 Subject: [PATCH 13/21] Cleanup --- Obsidian.ConsoleApp/Obsidian.ConsoleApp.csproj | 10 +++++++++- .../Plugins/PluginProviders/PackedPluginProvider.cs | 10 ++-------- Obsidian/Utilities/ServerConfiguration.cs | 2 +- SamplePlugin/SamplePlugin.csproj | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Obsidian.ConsoleApp/Obsidian.ConsoleApp.csproj b/Obsidian.ConsoleApp/Obsidian.ConsoleApp.csproj index b3ccf86ef..42dd32019 100644 --- a/Obsidian.ConsoleApp/Obsidian.ConsoleApp.csproj +++ b/Obsidian.ConsoleApp/Obsidian.ConsoleApp.csproj @@ -9,6 +9,12 @@ en + + + + + + @@ -18,7 +24,9 @@ - + + PreserveNewest + diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index b2e148a6c..daf2e97a3 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -1,9 +1,6 @@ using Microsoft.Extensions.Logging; using Obsidian.API.Plugins; using Org.BouncyCastle.Crypto; -using Org.BouncyCastle.Crypto.Utilities; -using Org.BouncyCastle.Pqc.Crypto.SphincsPlus; -using Org.BouncyCastle.Security; using System.Collections.Frozen; using System.IO; using System.Reflection; @@ -13,9 +10,6 @@ namespace Obsidian.Plugins.PluginProviders; public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger logger) { - private const int SignatureLength = 384; - private const int HashLength = 20; - private readonly PluginManager pluginManager = pluginManager; private readonly ILogger logger = logger; @@ -31,8 +25,8 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo //TODO save api version somewhere var apiVersion = reader.ReadString(); - var hash = reader.ReadBytes(HashLength); - var signature = reader.ReadBytes(SignatureLength); + var hash = reader.ReadBytes(SHA384.HashSizeInBytes); + var signature = reader.ReadBytes(SHA384.HashSizeInBits); var dataLength = reader.ReadInt32(); var curPos = fs.Position; diff --git a/Obsidian/Utilities/ServerConfiguration.cs b/Obsidian/Utilities/ServerConfiguration.cs index 5d9ad9d10..6cc49c0f4 100644 --- a/Obsidian/Utilities/ServerConfiguration.cs +++ b/Obsidian/Utilities/ServerConfiguration.cs @@ -19,7 +19,7 @@ public sealed class ServerConfiguration : IServerConfiguration public bool AllowOperatorRequests { get; set; } = true; - public bool AllowUntrustedPlugins { get; set; } = true; + public bool AllowUntrustedPlugins { get; set; } /// /// If true, each login/client gets a random username where multiple connections from the same host will be allowed. diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index f72d43113..74c2718ab 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -21,7 +21,7 @@ runtime - + From 380547951e571cac364568ad5e8d35ad7b4e7eea Mon Sep 17 00:00:00 2001 From: Tides Date: Thu, 7 Mar 2024 01:12:07 -0500 Subject: [PATCH 14/21] Add boolean check for signature --- .../_Interfaces/IServerConfiguration.cs | 2 +- Obsidian/Plugins/PluginManager.cs | 1 - .../PluginProviders/PackedPluginProvider.cs | 20 ++++++++++++------- SamplePlugin/SamplePlugin.csproj | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Obsidian.API/_Interfaces/IServerConfiguration.cs b/Obsidian.API/_Interfaces/IServerConfiguration.cs index 2b279cbed..af5bc68e9 100644 --- a/Obsidian.API/_Interfaces/IServerConfiguration.cs +++ b/Obsidian.API/_Interfaces/IServerConfiguration.cs @@ -14,7 +14,7 @@ public interface IServerConfiguration /// /// Determines where or not the server should load plugins that don't have a valid signature. /// - public bool AllowUntrustedPlugins { get; set; } + public bool AllowUntrustedPlugins { get; set; } = true; /// /// Allows the server to advertise itself as a LAN server to devices on your network. diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 5d49f1b42..3478fbfbb 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -115,7 +115,6 @@ public async Task LoadPluginsAsync() waitingForDepend.Remove(canLoad); } - if (pluginContainer.Plugin is null) waitingForDepend.Add(pluginContainer); } diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index daf2e97a3..9b0942d86 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -26,13 +26,16 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo var apiVersion = reader.ReadString(); var hash = reader.ReadBytes(SHA384.HashSizeInBytes); - var signature = reader.ReadBytes(SHA384.HashSizeInBits); + var isSigned = reader.ReadBoolean(); + + byte[]? signature = isSigned ? reader.ReadBytes(SHA384.HashSizeInBits) : null; + var dataLength = reader.ReadInt32(); var curPos = fs.Position; //Don't load untrusted plugins - var isSigValid = await this.TryValidatePluginAsync(fs, hash, signature, path); + var isSigValid = await this.TryValidatePluginAsync(fs, hash, path, isSigned, signature); if (!isSigValid) return null; @@ -113,7 +116,7 @@ internal PluginContainer HandlePlugin(PluginContainer pluginContainer, Assembly /// Verifies the file hash and tries to validate the signature /// /// - private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, byte[] signature, string path) + private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, string path, bool isSigned, byte[]? signature = null) { using (var sha384 = SHA384.Create()) { @@ -126,19 +129,22 @@ private async Task TryValidatePluginAsync(FileStream fs, byte[] hash, byte } } - var deformatter = new RSAPKCS1SignatureDeformatter(); - deformatter.SetHashAlgorithm("SHA384"); - var isSigValid = true; if (!this.pluginManager.server.Configuration.AllowUntrustedPlugins) { + if (!isSigned) + return false; + + var deformatter = new RSAPKCS1SignatureDeformatter(); + deformatter.SetHashAlgorithm("SHA384"); + using var rsa = RSA.Create(); foreach (var rsaParameter in this.pluginManager.AcceptedKeys) { rsa.ImportParameters(rsaParameter); deformatter.SetKey(rsa); - isSigValid = deformatter.VerifySignature(hash, signature); + isSigValid = deformatter.VerifySignature(hash, signature!); if (isSigValid) break; diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 74c2718ab..1e40e82c3 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -21,7 +21,7 @@ runtime - + From 752e4b0c6e66fc016b85d13d5dc1890b929ceb9c Mon Sep 17 00:00:00 2001 From: Tides Date: Sun, 17 Mar 2024 20:25:20 -0400 Subject: [PATCH 15/21] Add dependency when loaded --- Obsidian/Plugins/PluginFileEntry.cs | 1 - Obsidian/Plugins/PluginManager.cs | 4 ++++ Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs | 2 +- SamplePlugin/SamplePlugin.csproj | 1 - 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Obsidian/Plugins/PluginFileEntry.cs b/Obsidian/Plugins/PluginFileEntry.cs index eeaa95e6a..77556b391 100644 --- a/Obsidian/Plugins/PluginFileEntry.cs +++ b/Obsidian/Plugins/PluginFileEntry.cs @@ -1,5 +1,4 @@ using Org.BouncyCastle.Crypto; -using System.Buffers; using System.IO; using System.IO.Compression; diff --git a/Obsidian/Plugins/PluginManager.cs b/Obsidian/Plugins/PluginManager.cs index 3478fbfbb..98c6a71f2 100644 --- a/Obsidian/Plugins/PluginManager.cs +++ b/Obsidian/Plugins/PluginManager.cs @@ -110,6 +110,10 @@ public async Task LoadPluginsAsync() foreach (var canLoad in waitingForDepend.Where(x => x.IsDependency(pluginContainer.Info.Id)).ToList()) { packedPluginProvider.InitializePlugin(canLoad); + + //Add dependency to plugin + canLoad.LoadContext.AddDependency(pluginContainer.LoadContext); + this.HandlePlugin(canLoad); waitingForDepend.Remove(canLoad); diff --git a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs index 9b0942d86..4ac80ea1a 100644 --- a/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs +++ b/Obsidian/Plugins/PluginProviders/PackedPluginProvider.cs @@ -48,7 +48,7 @@ public sealed class PackedPluginProvider(PluginManager pluginManager, ILogger lo var entries = await this.InitializeEntriesAsync(reader, fs); - var partialContainer = BuildPartialContainer(loadContext, path, entries, isSigValid); + var partialContainer = this.BuildPartialContainer(loadContext, path, entries, isSigValid); //Can't load until those plugins are loaded if (partialContainer.Info.Dependencies.Any(x => x.Required && !this.pluginManager.Plugins.Any(d => d.Info.Id == x.Id))) diff --git a/SamplePlugin/SamplePlugin.csproj b/SamplePlugin/SamplePlugin.csproj index 1e40e82c3..be1bab1ca 100644 --- a/SamplePlugin/SamplePlugin.csproj +++ b/SamplePlugin/SamplePlugin.csproj @@ -36,7 +36,6 @@ PreserveNewest -