From 9da9556147a5a8a7c2cd10d104071b30be70ea72 Mon Sep 17 00:00:00 2001 From: Jens Melgaard <admin@it-links.dk> Date: Tue, 28 Nov 2023 21:49:30 +0100 Subject: [PATCH] Minimal working index contexts, but need a better solution for storage. --- .../LuceneIndexContextTest.cs | 41 ++++-- .../LuceneIndexContext.cs | 132 +++++++++++++----- .../Searching/LuceneJsonMultiIndexSearcher.cs | 5 +- .../IndexQueryParserExtensions.cs | 8 +- .../Configuration/JsonIndexConfiguration.cs | 2 +- src/DotJEM.Json.Index2/IJsonIndex.cs | 7 +- 6 files changed, 136 insertions(+), 59 deletions(-) diff --git a/src/DotJEM.Json.Index2.Contexts.Test/LuceneIndexContextTest.cs b/src/DotJEM.Json.Index2.Contexts.Test/LuceneIndexContextTest.cs index 70b7f34..a3184f6 100644 --- a/src/DotJEM.Json.Index2.Contexts.Test/LuceneIndexContextTest.cs +++ b/src/DotJEM.Json.Index2.Contexts.Test/LuceneIndexContextTest.cs @@ -20,20 +20,36 @@ public async Task SayHello_ReturnsHello() .ByDefault(x => x .UsingMemmoryStorage() .WithAnalyzer(cfg => new StandardAnalyzer(cfg.Version)) - .WithFieldResolver(new FieldResolver("uuid", "type")) - .Build()); - builder - .For("IndexName", x => x - .UsingMemmoryStorage() - .WithAnalyzer(cfg => new StandardAnalyzer(cfg.Version)) - .WithFieldResolver(new FieldResolver("uuid", "type")) - .Build()); + .WithFieldResolver(new FieldResolver("uuid", "type"))); + //builder + // .For("IndexName", x => x + // .UsingMemmoryStorage() + // .WithAnalyzer(cfg => new StandardAnalyzer(cfg.Version)) + // .WithFieldResolver(new FieldResolver("uuid", "type"))); + IJsonIndexContext context = builder.Build(); - context.Open("IndexName"); + context.Open("index1"); + + IJsonIndex index = context.Open("index1"); + IJsonIndex index2 = context.Open("index2"); + + WriteToIndex(index); + WriteToIndex(index2); + + IJsonIndexSearcher? searcher = index.CreateSearcher(); + IJsonIndexSearcher? searcher2 = index2.CreateSearcher(); + + Assert.That(searcher.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(5)); + Assert.That(searcher2.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(5)); - IJsonIndex index = context.Open("IndexName"); + IJsonIndexSearcher? combined = context.CreateSearcher(); + Assert.That(combined.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(10)); + + } + private static void WriteToIndex(IJsonIndex index) + { IJsonIndexWriter writer = index.CreateWriter(); writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); @@ -41,10 +57,5 @@ public async Task SayHello_ReturnsHello() writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); writer.Commit(); - - IJsonIndexSearcher? searcher = index.CreateSearcher(); - int count = searcher.Search(new TermQuery(new Term("type", "car"))).Count(); - //int count = searcher.Search(new MatchAllDocsQuery()).Count(); - Assert.AreEqual(5, count); } } \ No newline at end of file diff --git a/src/DotJEM.Json.Index2.Contexts/LuceneIndexContext.cs b/src/DotJEM.Json.Index2.Contexts/LuceneIndexContext.cs index 865db42..75965b3 100644 --- a/src/DotJEM.Json.Index2.Contexts/LuceneIndexContext.cs +++ b/src/DotJEM.Json.Index2.Contexts/LuceneIndexContext.cs @@ -1,76 +1,84 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; using DotJEM.Json.Index2.Configuration; using DotJEM.Json.Index2.Contexts.Searching; using DotJEM.Json.Index2.Contexts.Storage; using DotJEM.Json.Index2.Searching; +using DotJEM.Json.Index2.Storage; +using Lucene.Net.Util; namespace DotJEM.Json.Index2.Contexts; -public interface IJsonIndexContext : IJsonIndexSearcherProvider +public interface IJsonIndexContext { - IJsonIndex Open(string name); + IJsonIndex Open(string index); + IJsonIndex Open(string group, string index); + IJsonIndexSearcher CreateSearcher(); + IJsonIndexSearcher CreateSearcher(string group); } public class JsonIndexContext : IJsonIndexContext { private readonly IJsonIndexFactory factory; - private readonly ConcurrentDictionary<string, IJsonIndex> indices = new ConcurrentDictionary<string, IJsonIndex>(); - - //public IServiceResolver Services { get; } - //public JsonIndexContext(IServiceCollection services = null) - // : this(new LuceneIndexContextBuilder(), services) { } - - //public JsonIndexContext(string path, IServiceCollection services = null) - // : this(new LuceneIndexContextBuilder(path), services) { } + private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, IJsonIndex>> indices = new (); public JsonIndexContext(IJsonIndexFactory factory) { this.factory = factory ?? throw new ArgumentNullException(nameof(factory)); - //this.Services = resolver ?? throw new ArgumentNullException(nameof(resolver)); } - public IJsonIndex Open(string name) + public IJsonIndex Open(string index) + => Open("*", index); + + public IJsonIndex Open(string group, string index) { - return indices.GetOrAdd(name, factory.Create); + ConcurrentDictionary<string, IJsonIndex> map = indices.GetOrAdd(group, s => new ConcurrentDictionary<string, IJsonIndex>()); + return map.GetOrAdd(index, factory.Create); } public IJsonIndexSearcher CreateSearcher() + => CreateSearcher("*"); + + public IJsonIndexSearcher CreateSearcher(string group) { - return new LuceneJsonMultiIndexSearcher(indices.Values); + ConcurrentDictionary<string, IJsonIndex> map = indices.GetOrAdd(group, s => new ConcurrentDictionary<string, IJsonIndex>()); + return new LuceneJsonMultiIndexSearcher(map.Values); } } public interface IJsonIndexContextBuilder { - IJsonIndexContextBuilder ByDefault(Func<IJsonIndexBuilder, IJsonIndex> defaultConfig); - IJsonIndexContextBuilder For(string name, Func<IJsonIndexBuilder, IJsonIndex> defaultConfig); + IJsonIndexContextBuilder ByDefault(Action<IJsonIndexBuilder> configure); + IJsonIndexContextBuilder For(string group, Action<IJsonIndexBuilder> configure); IJsonIndexContext Build(); } public class JsonIndexContextBuilder : IJsonIndexContextBuilder { - private readonly ConcurrentDictionary<string, Func<IJsonIndexBuilder, IJsonIndex>> configurators = new(); - public IJsonIndexContextBuilder ByDefault(Func<IJsonIndexBuilder, IJsonIndex> defaultConfig) + private readonly ConcurrentDictionary<string, Action<IJsonIndexBuilder>> configurators = new(); + + public IJsonIndexContextBuilder ByDefault(Action<IJsonIndexBuilder> configure) { - configurators.AddOrUpdate("*", s => defaultConfig, (s, func) => defaultConfig); + configurators.AddOrUpdate("*", s => configure, (s, func) => configure); return this; } - public IJsonIndexContextBuilder For(string name, Func<IJsonIndexBuilder, IJsonIndex> defaultConfig) + public IJsonIndexContextBuilder For(string group, Action<IJsonIndexBuilder> configure) { - if (name == null) throw new ArgumentNullException(nameof(name)); - if (defaultConfig == null) throw new ArgumentNullException(nameof(defaultConfig)); - if (name is "*" or "") throw new ArgumentException("Invalid name for an index.", nameof(name)); + if (group == null) throw new ArgumentNullException(nameof(group)); + if (configure == null) throw new ArgumentNullException(nameof(configure)); + if (group is "*" or "") throw new ArgumentException("Invalid name for an index.", nameof(group)); - configurators.AddOrUpdate(name, s => defaultConfig, (s, func) => defaultConfig); + configurators.AddOrUpdate(group, s => configure, (s, func) => configure); return this; } public IJsonIndexContext Build() { - return new JsonIndexContext(new JsonIndexFactory(new Dictionary<string, Func<IJsonIndexBuilder, IJsonIndex>>(configurators))); + return new JsonIndexContext(new JsonIndexFactory(new Dictionary<string, Action<IJsonIndexBuilder>>(configurators))); } } @@ -81,21 +89,79 @@ public interface IJsonIndexFactory public class JsonIndexFactory : IJsonIndexFactory { - private readonly IReadOnlyDictionary<string, Func<IJsonIndexBuilder, IJsonIndex>> configurators; + private readonly IReadOnlyDictionary<string, Action<IJsonIndexBuilder>> configurators; + private readonly ConcurrentDictionary<string, Entry> configurations = new(); - public JsonIndexFactory(IReadOnlyDictionary<string, Func<IJsonIndexBuilder, IJsonIndex>> configurators) + public JsonIndexFactory(IReadOnlyDictionary<string,Action<IJsonIndexBuilder>> configurators) { this.configurators = configurators; } public IJsonIndex Create(string name) { - if (configurators.TryGetValue(name, out Func<IJsonIndexBuilder, IJsonIndex> func)) - return func(new JsonIndexBuilder(name)); + Entry entry = configurations.GetOrAdd(name, CreateConfiguration); + return new JsonIndex(entry.StorageProvider, entry.Configuration); + } + + private Entry CreateConfiguration(string group) + { + if (!TryGetConfigurator(group, out Action<IJsonIndexBuilder> configure)) + throw new ArgumentException("No configurators found for the given group and no default configurators provided.", nameof(group)); + + JsonIndexBuilderForContexts builder = new JsonIndexBuilderForContexts(); + configure(builder); + + //TODO: StorageProviderFactory instead. + return new Entry(builder.StorageProvider, builder.BuildConfiguration()); + } + + private bool TryGetConfigurator(string group, out Action<IJsonIndexBuilder> cfg) + { + return configurators.TryGetValue(group, out cfg) + || configurators.TryGetValue("*", out cfg); + } + + private record struct Entry(IIndexStorageProvider StorageProvider, IJsonIndexConfiguration Configuration); +} + +public class JsonIndexBuilderForContexts : IJsonIndexBuilder +{ + public string Name { get; } = Guid.NewGuid().ToString("D"); + public IIndexStorageProvider StorageProvider { get; private set; } = new RamIndexStorageProvider(); + private readonly Dictionary<Type, Func<IJsonIndexConfiguration, object>> factories = new(); + + public IJsonIndexBuilder UsingStorage(IIndexStorageProvider storageProvider) + { + this.StorageProvider = storageProvider; + return this; + } + + public IJsonIndexBuilder WithService<TService>(bool replace, Func<IJsonIndexConfiguration, TService> factory) + { + if (factory == null) throw new ArgumentNullException(nameof(factory)); + if (replace) + factories[typeof(TService)] = cfg => factory(cfg); + else + { + +#if NETSTANDARD2_0 + if(!factories.ContainsKey(typeof(TService))) + factories.Add(typeof(TService), cfg => factory(cfg)); +#else + factories.TryAdd(typeof(TService), cfg => factory(cfg)); +#endif + } + return this; + } - if(configurators.TryGetValue("*", out func)) - return func(new JsonIndexBuilder(name)); + public IJsonIndex Build() + { + return new JsonIndex(StorageProvider, BuildConfiguration()); + } - throw new InvalidOperationException(""); + public IJsonIndexConfiguration BuildConfiguration() + { + return new JsonIndexConfiguration(LuceneVersion.LUCENE_48, factories + .Select(pair => new ServiceDescriptor(pair.Key, pair.Value))); } -} \ No newline at end of file +} diff --git a/src/DotJEM.Json.Index2.Contexts/Searching/LuceneJsonMultiIndexSearcher.cs b/src/DotJEM.Json.Index2.Contexts/Searching/LuceneJsonMultiIndexSearcher.cs index e9d8368..089789c 100644 --- a/src/DotJEM.Json.Index2.Contexts/Searching/LuceneJsonMultiIndexSearcher.cs +++ b/src/DotJEM.Json.Index2.Contexts/Searching/LuceneJsonMultiIndexSearcher.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using DotJEM.Json.Index2.Configuration; using DotJEM.Json.Index2.Results; using DotJEM.Json.Index2.Searching; using DotJEM.Json.Index2.Util; @@ -15,15 +16,17 @@ public class LuceneJsonMultiIndexSearcher : Disposable, IJsonIndexSearcher public IInfoStream InfoStream { get; } = new InfoStream<LuceneJsonMultiIndexSearcher>(); private readonly IJsonIndex[] indicies; + private readonly IJsonIndexConfiguration configuration; public LuceneJsonMultiIndexSearcher(IEnumerable<IJsonIndex> indicies) { this.indicies = indicies.ToArray(); + this.configuration = this.indicies.First().Configuration; } public ISearch Search(Query query) { - return new Search(new MultiIndexJsonSearcherManager(indicies, null), query); + return new Search(new MultiIndexJsonSearcherManager(indicies, configuration.Serializer), query); } } } \ No newline at end of file diff --git a/src/DotJEM.Json.Index2.QueryParsers/IndexQueryParserExtensions.cs b/src/DotJEM.Json.Index2.QueryParsers/IndexQueryParserExtensions.cs index 9b34cd5..5183ce9 100644 --- a/src/DotJEM.Json.Index2.QueryParsers/IndexQueryParserExtensions.cs +++ b/src/DotJEM.Json.Index2.QueryParsers/IndexQueryParserExtensions.cs @@ -14,21 +14,21 @@ public static IJsonIndexBuilder WithSimplifiedLuceneQueryParser(this IJsonIndexB public static ISearch Search(this IJsonIndexSearcher self, string query) { - ILuceneQueryParser parser = self.Index.ResolveParser(); + ILuceneQueryParser parser = self.Index.Configuration.ResolveParser(); LuceneQueryInfo queryInfo = parser.Parse(query); return self.Search(queryInfo.Query).OrderBy(queryInfo.Sort); } public static ISearch Search(this IJsonIndex self, string query) { - ILuceneQueryParser parser = self.ResolveParser(); + ILuceneQueryParser parser = self.Configuration.ResolveParser(); LuceneQueryInfo queryInfo = parser.Parse(query); return self.CreateSearcher().Search(queryInfo.Query).OrderBy(queryInfo.Sort); } - private static ILuceneQueryParser ResolveParser(this IJsonIndex self) + private static ILuceneQueryParser ResolveParser(this IJsonIndexConfiguration self) { - return self.Configuration.Get<ILuceneQueryParser>() ?? throw new Exception("Query parser not configured."); + return self.Get<ILuceneQueryParser>() ?? throw new Exception("Query parser not configured."); } } \ No newline at end of file diff --git a/src/DotJEM.Json.Index2/Configuration/JsonIndexConfiguration.cs b/src/DotJEM.Json.Index2/Configuration/JsonIndexConfiguration.cs index f35bc70..b418433 100644 --- a/src/DotJEM.Json.Index2/Configuration/JsonIndexConfiguration.cs +++ b/src/DotJEM.Json.Index2/Configuration/JsonIndexConfiguration.cs @@ -33,7 +33,7 @@ public JsonIndexConfiguration() Serializer = new GZipJsonDocumentSerialier(); } - internal JsonIndexConfiguration(LuceneVersion version, IEnumerable<ServiceDescriptor> services) + public JsonIndexConfiguration(LuceneVersion version, IEnumerable<ServiceDescriptor> services) { Services = new ServiceCollection(this, services); Version = version; diff --git a/src/DotJEM.Json.Index2/IJsonIndex.cs b/src/DotJEM.Json.Index2/IJsonIndex.cs index ba318dd..5924838 100644 --- a/src/DotJEM.Json.Index2/IJsonIndex.cs +++ b/src/DotJEM.Json.Index2/IJsonIndex.cs @@ -15,12 +15,8 @@ namespace DotJEM.Json.Index2; -public interface IJsonIndexSearcherProvider -{ - IJsonIndexSearcher CreateSearcher(); -} -public interface IJsonIndex : IJsonIndexSearcherProvider +public interface IJsonIndex { IInfoStream InfoStream { get; } IJsonIndexStorageManager Storage { get; } @@ -28,6 +24,7 @@ public interface IJsonIndex : IJsonIndexSearcherProvider IIndexWriterManager WriterManager { get; } IIndexSearcherManager SearcherManager { get; } IJsonIndexWriter CreateWriter(); + IJsonIndexSearcher CreateSearcher(); void Close(); }