diff --git a/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj b/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj index 14c8960..c656b1a 100644 --- a/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj +++ b/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj @@ -23,7 +23,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj b/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj index 8ae61c2..2d38d44 100644 --- a/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj +++ b/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj b/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj index 7991abc..253f8c9 100644 --- a/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj +++ b/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj @@ -7,7 +7,7 @@ - + @@ -16,7 +16,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs b/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs index d916a74..db10c39 100644 --- a/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs +++ b/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs @@ -128,11 +128,6 @@ public async Task ResetIndexAsync() { await jsonDocumentSource.StopAsync().ConfigureAwait(false); index.Storage.Delete(); - //TODO: Force a commit after delete to see if that helps? - using (var lease = index.WriterManager.Lease()) - { - lease.Value.Commit(); - } await jsonDocumentSource.ResetAsync().ConfigureAwait(false); await jsonDocumentSource.StartAsync().ConfigureAwait(false); } diff --git a/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs b/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs index 7c47dd5..55d33b2 100644 --- a/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs +++ b/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs @@ -6,6 +6,7 @@ using DotJEM.Json.Index2.Snapshots; using DotJEM.ObservableExtensions.InfoStreams; using DotJEM.Web.Scheduler; +using Lucene.Net.Search; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -41,6 +42,8 @@ public class JsonIndexSnapshotManager : IJsonIndexSnapshotManager { private readonly IJsonIndex index; private readonly ISnapshotStrategy strategy; + //TODO: This binds us to a concrete idea (even though this is an interface). + // instead it would be preferred if we could provide an abstraction for what is needed. private readonly IWebTaskScheduler scheduler; private readonly IInfoStream infoStream = new InfoStream(); diff --git a/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs b/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs index 4e583d5..58c759a 100644 --- a/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs +++ b/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs @@ -7,6 +7,7 @@ using DotJEM.Json.Index2.Documents; using DotJEM.Json.Index2.Documents.Info; using DotJEM.Json.Index2.IO; +using DotJEM.Json.Index2.Leases; using DotJEM.ObservableExtensions.InfoStreams; using Lucene.Net.Index; using Newtonsoft.Json.Linq; @@ -154,6 +155,7 @@ private void Commit() try { lease.Value.Commit(); + } catch (Exception e) { diff --git a/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj b/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj index 3d9df39..e8dc90e 100644 --- a/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj +++ b/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj @@ -13,7 +13,7 @@ - + @@ -24,7 +24,7 @@ - + diff --git a/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj b/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj index 1a0b4e3..ff11b7f 100644 --- a/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj +++ b/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj @@ -20,7 +20,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj b/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj index 2b975bb..5247129 100644 --- a/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj +++ b/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs b/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs index cc7b1bc..d1ab3ab 100644 --- a/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs +++ b/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using DotJEM.Json.Index2.IO; +using DotJEM.Json.Index2.Leases; using DotJEM.Json.Index2.Snapshots.Streams; using Lucene.Net.Index; using Lucene.Net.Store; diff --git a/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj b/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj index 63cb127..6bcb848 100644 --- a/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj +++ b/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj @@ -22,7 +22,7 @@ - + diff --git a/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs b/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs index 9251c5c..e118b60 100644 --- a/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs +++ b/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs @@ -34,6 +34,7 @@ public async Task Create_AddsDocument() //int count = searcher.Search(new MatchAllDocsQuery()).Count(); Assert.AreEqual(5, count); } + [Test] public async Task SayHello_ReturnsHello() { @@ -83,4 +84,26 @@ public async Task Search_Booleans() int count = searcher.Search(new TermQuery(new Term("inStock", "true"))).Count(); Assert.AreEqual(3, count); } + [Test] + public async Task FindBeforeCommit_AddsDocument() + { + IJsonIndex index = new JsonIndexBuilder("myIndex") + .UsingMemmoryStorage() + .WithAnalyzer(cfg => new StandardAnalyzer(cfg.Version)) + .WithFieldResolver(new FieldResolver("uuid", "type")) + .Build(); + + IJsonIndexWriter writer = index.CreateWriter(); + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + + IJsonIndexSearcher? searcher = index.CreateSearcher(); + Assert.That(searcher.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(5)); + + writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" })); + Assert.That(searcher.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(6)); + } } \ No newline at end of file diff --git a/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj b/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj index 153cda4..93efaa3 100644 --- a/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj +++ b/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj @@ -44,9 +44,9 @@ - - - + + + diff --git a/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs b/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs index af0b8df..1b6031f 100644 --- a/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs +++ b/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using DotJEM.Json.Index2.Documents; +using DotJEM.Json.Index2.Leases; using DotJEM.Json.Index2.Util; using Lucene.Net.Documents; using Lucene.Net.Index; diff --git a/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs b/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs index 60ad7d3..79cc124 100644 --- a/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs +++ b/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs @@ -4,25 +4,24 @@ using System.Diagnostics; using System.Linq; using System.Threading; +using System.Threading.Tasks; using DotJEM.Json.Index2.Configuration; +using DotJEM.Json.Index2.Leases; using DotJEM.Json.Index2.Util; using Lucene.Net.Analysis; using Lucene.Net.Index; +using Lucene.Net.Search; namespace DotJEM.Json.Index2.IO; public interface IIndexWriterManager : IDisposable { event EventHandler OnClose; + ILease Lease(); void Close(); } -public interface ILease : IDisposable -{ - T Value { get; } - bool IsExpired { get; } -} public class IndexWriterManager : Disposable, IIndexWriterManager { @@ -31,8 +30,9 @@ public class IndexWriterManager : Disposable, IIndexWriterManager private readonly IJsonIndex index; private volatile IndexWriter writer; private readonly object writerPadLock = new(); - private readonly object leasesPadLock = new(); + private readonly LeaseManager leaseManager = new(); + //TODO: With leases, this should not be needed. public event EventHandler OnClose; private IndexWriter Writer @@ -47,40 +47,13 @@ private IndexWriter Writer if (writer != null) return writer; - try - { - return writer = Open(index); - } - catch (Exception e) - { - Debug.WriteLine("ACTIVE LEASES: " +leases.Count); - throw; - } + return writer = Open(index); } } } - private readonly List leases = new List(); - - public ILease Lease() - { - //TODO: Optimizied collection for this. - TimeLimitedIndexWriterLease lease = new(this, OnReturned); - lock (leasesPadLock) - { - leases.Add(lease); - } - return lease; - } - - private void OnReturned(TimeLimitedIndexWriterLease lease) - { - lock (leasesPadLock) - { - leases.Remove(lease); - } - } + public ILease Lease() => leaseManager.Create(Writer, TimeSpan.FromSeconds(3)); public IndexWriterManager(IJsonIndex index) { @@ -89,7 +62,6 @@ public IndexWriterManager(IJsonIndex index) private static IndexWriter Open(IJsonIndex index) { - Debug.WriteLine("OPEN WRITER"); IndexWriterConfig config = new(index.Configuration.Version, index.Configuration.Analyzer); config.RAMBufferSizeMB = DEFAULT_RAM_BUFFER_SIZE_MB; config.OpenMode = OpenMode.CREATE_OR_APPEND; @@ -103,28 +75,15 @@ public void Close() if (writer == null) return; - lock (leasesPadLock) + lock (writerPadLock) { - TimeLimitedIndexWriterLease[] leasesCopy = leases.ToArray(); - - Debug.WriteLine("ACTIVE LEASES: " + leasesCopy.Length); - leases.Clear(); - foreach (TimeLimitedIndexWriterLease lease in leasesCopy) - { - if (!lease.IsExpired) - lease.Wait(); - lease.Dispose(); - } - - lock (writerPadLock) - { - if (writer == null) - return; + leaseManager.RecallAll(); + if (writer == null) + return; - writer.Dispose(); - writer = null; - RaiseOnClose(); - } + writer.Dispose(); + writer = null; + RaiseOnClose(); } } @@ -140,63 +99,4 @@ protected virtual void RaiseOnClose() { OnClose?.Invoke(this, EventArgs.Empty); } - - private class TimeLimitedIndexWriterLease : Disposable, ILease - { - private readonly DateTime leaseTime = DateTime.Now; - private readonly Action onReturned; - private readonly IndexWriterManager manager; - public AutoResetEvent Handle { get; } = new(false); - - public IndexWriter Value - { - get - { - if (IsDisposed) - { - throw new ObjectDisposedException("Index writer lease has been returned or is expired."); - } - - if (IsExpired) - { - throw new LeaseExpiredException("Index writer lease has been returned or is expired."); - } - return manager.Writer; - } - } - - public bool IsExpired => (DateTime.Now - leaseTime > TimeSpan.FromSeconds(5)) || IsDisposed; - - public TimeLimitedIndexWriterLease(IndexWriterManager manager, Action onReturned) - { - this.onReturned = onReturned; - this.manager = manager; - } - - protected override void Dispose(bool disposing) - { - Handle.Set(); - onReturned(this); - } - - public void Wait() - { - if (IsExpired) - return; - - Handle.WaitOne(TimeSpan.FromSeconds(6) - (DateTime.Now - leaseTime)); - } - } - } - -public class LeaseExpiredException : Exception -{ - public LeaseExpiredException(string message) : base(message) - { - } - - public LeaseExpiredException(string message, Exception innerException) : base(message, innerException) - { - } -} \ No newline at end of file diff --git a/src/DotJEM.Json.Index2/Leases/Lease.cs b/src/DotJEM.Json.Index2/Leases/Lease.cs new file mode 100644 index 0000000..0df807c --- /dev/null +++ b/src/DotJEM.Json.Index2/Leases/Lease.cs @@ -0,0 +1,18 @@ +using System; + +namespace DotJEM.Json.Index2.Leases; + +public interface ILease : IDisposable +{ + event EventHandler Terminated; + + T Value { get; } + bool IsExpired { get; } + bool TryRenew(); +} + + +public interface ILessor +{ + ILease Lease(); +} diff --git a/src/DotJEM.Json.Index2/Leases/LeaseManager.cs b/src/DotJEM.Json.Index2/Leases/LeaseManager.cs new file mode 100644 index 0000000..d89bc9e --- /dev/null +++ b/src/DotJEM.Json.Index2/Leases/LeaseManager.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using DotJEM.Json.Index2.Util; + +namespace DotJEM.Json.Index2.Leases; + +public interface ILeaseManager +{ + int Count { get; } + ILease Create(T value); + ILease Create(T value, TimeSpan limit); + void RecallAll(); +} + +public class LeaseManager : ILeaseManager +{ + //TODO: Optimizied collection for this. + private readonly List> leases = new(); + private readonly object leasesPadLock = new(); + + public int Count => leases.Count; + + public ILease Create(T value) + { + return Add(new Lease(value, OnReturned)); + } + + public ILease Create(T value, TimeSpan limit) + { + return Add(new TimeLimitedLease(value, OnReturned, limit)); + } + + public void RecallAll() + { + lock (leasesPadLock) + { + IRecallableLease[] copy = leases.ToArray(); + leases.Clear(); + + foreach (IRecallableLease lease in copy) + lease.Terminate(); + } + } + + private ILease Add(IRecallableLease lease) + { + lock (leasesPadLock) + { + leases.Add(lease); + } + return lease; + } + + private void OnReturned(IRecallableLease obj) + { + lock (leasesPadLock) + { + leases.Remove(obj); + } + } + + private interface IRecallableLease : ILease + { + void Terminate(); + } + + private class Lease : Disposable, IRecallableLease + { + public event EventHandler Terminated; + + private readonly T value; + private readonly Action> onReturned; + + public bool IsExpired => IsDisposed; + + + public T Value + { + get + { + if (IsExpired) + { + throw new LeaseExpiredException("Lease is expired as it was returned."); + } + return value; + } + } + + public Lease(T value, Action> onReturned) + { + this.value = value; + this.onReturned = onReturned; + } + + public bool TryRenew() + { + return false; + } + + public void Terminate() + { + Terminated?.Invoke(this, EventArgs.Empty); + Dispose(); + } + + protected override void Dispose(bool disposing) + { + onReturned(this); + base.Dispose(disposing); + } + } + + private class TimeLimitedLease : Disposable, IRecallableLease + { + public event EventHandler Terminated; + + private readonly T value; + private readonly Action> onReturned; + private readonly TimeSpan timeLimit; + private readonly DateTime leaseTime = DateTime.Now; + private readonly AutoResetEvent handle = new(false); + + public bool IsExpired => (DateTime.Now - leaseTime > timeLimit) || IsDisposed; + + public T Value + { + get + { + if (IsExpired) + { + throw new LeaseExpiredException($"Lease is expired either because the time limit '{timeLimit}' has exceeded or the lease was returned."); + } + return value; + } + } + + public TimeLimitedLease(T value, Action> onReturned, TimeSpan timeLimit) + { + this.value = value; + this.onReturned = onReturned; + this.timeLimit = timeLimit; + } + public bool TryRenew() + { + return false; + } + + public void Terminate() + { + Terminated?.Invoke(this, EventArgs.Empty); + Wait(); + Dispose(); + } + + public void Wait() + { + if (IsExpired) + return; + + handle.WaitOne(TimeSpan.FromSeconds(6) - (DateTime.Now - leaseTime)); + } + + protected override void Dispose(bool disposing) + { + onReturned(this); + base.Dispose(disposing); + } + } +} +public class LeaseExpiredException : Exception +{ + public LeaseExpiredException(string message) : base(message) + { + } + + public LeaseExpiredException(string message, Exception innerException) : base(message, innerException) + { + } +} + +public class LeaseTerminatedException : Exception +{ + public LeaseTerminatedException(string message) : base(message) + { + } + + public LeaseTerminatedException(string message, Exception innerException) : base(message, innerException) + { + } +} \ No newline at end of file diff --git a/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs b/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs index e79b925..abd446a 100644 --- a/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs +++ b/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs @@ -1,4 +1,7 @@ -using Lucene.Net.Store; +using System; +using System.IO; +using Lucene.Net.Store; +using Directory = Lucene.Net.Store.Directory; namespace DotJEM.Json.Index2.Storage; @@ -32,5 +35,12 @@ public SimpleFsIndexStorageProvider(string path) public void Delete() { + //TODO: For now. But maybe there is cases where this actually makes sense to always have. + DirectoryInfo dir = new DirectoryInfo(path); + foreach (FileInfo file in dir.EnumerateFiles()) + file.Delete(); + + foreach (DirectoryInfo directory in dir.EnumerateDirectories()) + directory.Delete(true); } } \ No newline at end of file diff --git a/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs b/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs index 9402776..cf2738a 100644 --- a/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs +++ b/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs @@ -1,6 +1,7 @@ using System; using System.Text.RegularExpressions; using DotJEM.Json.Index2.IO; +using DotJEM.Json.Index2.Leases; using DotJEM.Json.Index2.Searching; using Lucene.Net.Index; using Lucene.Net.Store; @@ -23,6 +24,7 @@ public class JsonIndexStorageManager: IJsonIndexStorageManager private readonly IIndexStorageProvider provider; private readonly object padlock = new (); private volatile Directory directory; + private readonly LeaseManager leaseManager = new(); private readonly Lazy writerManager; private readonly Lazy searcherManager; @@ -40,7 +42,6 @@ public Directory Directory { if (directory != null) return directory; - return directory = provider.Get(); } } @@ -49,8 +50,8 @@ public Directory Directory public JsonIndexStorageManager(IJsonIndex index, IIndexStorageProvider provider) { this.provider = provider; - this.writerManager = new Lazy(()=> new IndexWriterManager(index)); - this.searcherManager = new Lazy(()=> new IndexSearcherManager(WriterManager, index.Configuration.Serializer)); + this.writerManager = new(()=> new IndexWriterManager(index)); + this.searcherManager = new(()=> new IndexSearcherManager(WriterManager, index.Configuration.Serializer)); } public void Unlock() @@ -80,8 +81,6 @@ public void Delete() foreach (string file in directory.ListAll()) directory.DeleteFile(file); provider.Delete(); - - } } } \ No newline at end of file diff --git a/src/Stress/StressTester/Program.cs b/src/Stress/StressTester/Program.cs index ac4cf1b..06f1947 100644 --- a/src/Stress/StressTester/Program.cs +++ b/src/Stress/StressTester/Program.cs @@ -11,6 +11,7 @@ using DotJEM.Json.Index2.Management.Info; using DotJEM.Json.Index2.Management.Snapshots; using DotJEM.Json.Index2.Management.Snapshots.Zip; +using DotJEM.Json.Index2.Management.Source; using DotJEM.Json.Index2.Management.Tracking; using DotJEM.Json.Index2.Management.Writer; using DotJEM.Json.Index2.Searching; @@ -29,8 +30,9 @@ //TraceSource trace; +IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=SSN3DB;Integrated Security=True"); //IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=nsw;Integrated Security=True"); -IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True"); +//IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True"); storage.Configure.MapField(JsonField.Id, "id"); storage.Configure.MapField(JsonField.ContentType, "contentType"); storage.Configure.MapField(JsonField.Version, "$version"); @@ -38,19 +40,19 @@ storage.Configure.MapField(JsonField.Updated, "$updated"); storage.Configure.MapField(JsonField.SchemaVersion, "$schemaVersion"); -StressDataGenerator generator = new StressDataGenerator( - storage.Area("Settings"), - storage.Area("Queue"), - storage.Area("Recipes"), - storage.Area("Animals"), - storage.Area("Games"), - storage.Area("Players"), - storage.Area("Planets"), - storage.Area("Universe"), - storage.Area("Trashcan") -); -Task genTask = generator.StartAsync(); -await Task.Delay(2000); +//StressDataGenerator generator = new StressDataGenerator( +// storage.Area("Settings"), +// storage.Area("Queue"), +// storage.Area("Recipes"), +// storage.Area("Animals"), +// storage.Area("Games"), +// storage.Area("Players"), +// storage.Area("Planets"), +// storage.Area("Universe"), +// storage.Area("Trashcan") +//); +//Task genTask = generator.StartAsync(); +//await Task.Delay(2000); if (Directory.Exists(@".\app_data\index")) Directory.Delete(@".\app_data\index", true); @@ -67,12 +69,26 @@ string[] areas = new[] { "content", "settings", "diagnostic", "emsaqueue", "statistic" }; IWebTaskScheduler scheduler = new WebTaskScheduler(); +IJsonDocumentSource jsonStorageDocumentSource = new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler, areas)); IJsonIndexManager jsonIndexManager = new JsonIndexManager( - new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler,areas)), + jsonStorageDocumentSource, new JsonIndexSnapshotManager(index, new ZipSnapshotStrategy(".\\app_data\\snapshots"), scheduler, "30m"), index ); + +DateTime start = DateTime.Now; +TimeSpan completedAfter = TimeSpan.Zero; +jsonStorageDocumentSource.Initialized + .Where(b => b) + .FirstAsync(b => + { + completedAfter = DateTime.Now - start; + Console.WriteLine($"Done after: {completedAfter:g}"); + return b; + }); + + Task run = Task.WhenAll( jsonIndexManager.InfoStream.ForEachAsync(Reporter.CaptureInfo), jsonIndexManager.RunAsync() @@ -123,10 +139,10 @@ public static class Reporter { private static ITrackerState lastState; private static IInfoStreamEvent lastEvent; - private static DateTime lastReport = DateTime.Now; + private static DateTime lastReport = DateTime.Now.Subtract(TimeSpan.FromMinutes(1)); private static readonly Queue messages = new Queue(); - + private static long eventCounter = 0; public static void CaptureInfo(IInfoStreamEvent evt) { lock (messages) @@ -147,11 +163,11 @@ public static void CaptureInfo(IInfoStreamEvent evt) break; default: - lastEvent = evt; break; } - Report(); + if(eventCounter++ % 10000 == 0) + Console.Write('.'); } private static string CleanLine = new string(' ', Console.BufferWidth); @@ -160,7 +176,7 @@ public static void CaptureInfo(IInfoStreamEvent evt) public static void Report(bool force = false) { - if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(30)) + if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(5)) return; lastReport = DateTime.Now; int lines = nl.Matches(buffer.ToString()).Count; @@ -176,15 +192,11 @@ public static void Report(bool force = false) { msgs = messages.ToArray(); } - + //Console.WriteLine(lastEvent.Message); + buffer.AppendLine(lastState?.ToString()); foreach (string message in msgs) - { buffer.AppendLine(message); - } - buffer.AppendLine(); - //Console.WriteLine(lastEvent.Message); - buffer.AppendLine(lastState.ToString()); Console.WriteLine(buffer); } } diff --git a/src/Stress/StressTester/StressTester.csproj b/src/Stress/StressTester/StressTester.csproj index aa7b510..c3ae235 100644 --- a/src/Stress/StressTester/StressTester.csproj +++ b/src/Stress/StressTester/StressTester.csproj @@ -12,9 +12,9 @@ - - - + + +