diff --git a/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj b/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj
index 14c8960..c656b1a 100644
--- a/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj
+++ b/src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj
@@ -23,7 +23,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj b/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj
index 8ae61c2..2d38d44 100644
--- a/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj
+++ b/src/DotJEM.Json.Index2.Contexts/DotJEM.Json.Index2.Contexts.csproj
@@ -11,7 +11,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj b/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj
index 7991abc..253f8c9 100644
--- a/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj
+++ b/src/DotJEM.Json.Index2.Management/DotJEM.Json.Index2.Management.csproj
@@ -7,7 +7,7 @@
-
+
@@ -16,7 +16,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs b/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs
index d916a74..db10c39 100644
--- a/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs
+++ b/src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs
@@ -128,11 +128,6 @@ public async Task ResetIndexAsync()
{
await jsonDocumentSource.StopAsync().ConfigureAwait(false);
index.Storage.Delete();
- //TODO: Force a commit after delete to see if that helps?
- using (var lease = index.WriterManager.Lease())
- {
- lease.Value.Commit();
- }
await jsonDocumentSource.ResetAsync().ConfigureAwait(false);
await jsonDocumentSource.StartAsync().ConfigureAwait(false);
}
diff --git a/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs b/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs
index 7c47dd5..55d33b2 100644
--- a/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs
+++ b/src/DotJEM.Json.Index2.Management/Snapshots/IJsonIndexSnapshotManager.cs
@@ -6,6 +6,7 @@
using DotJEM.Json.Index2.Snapshots;
using DotJEM.ObservableExtensions.InfoStreams;
using DotJEM.Web.Scheduler;
+using Lucene.Net.Search;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
@@ -41,6 +42,8 @@ public class JsonIndexSnapshotManager : IJsonIndexSnapshotManager
{
private readonly IJsonIndex index;
private readonly ISnapshotStrategy strategy;
+ //TODO: This binds us to a concrete idea (even though this is an interface).
+ // instead it would be preferred if we could provide an abstraction for what is needed.
private readonly IWebTaskScheduler scheduler;
private readonly IInfoStream infoStream = new InfoStream();
diff --git a/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs b/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs
index 4e583d5..58c759a 100644
--- a/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs
+++ b/src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs
@@ -7,6 +7,7 @@
using DotJEM.Json.Index2.Documents;
using DotJEM.Json.Index2.Documents.Info;
using DotJEM.Json.Index2.IO;
+using DotJEM.Json.Index2.Leases;
using DotJEM.ObservableExtensions.InfoStreams;
using Lucene.Net.Index;
using Newtonsoft.Json.Linq;
@@ -154,6 +155,7 @@ private void Commit()
try
{
lease.Value.Commit();
+
}
catch (Exception e)
{
diff --git a/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj b/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj
index 3d9df39..e8dc90e 100644
--- a/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj
+++ b/src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj
@@ -13,7 +13,7 @@
-
+
@@ -24,7 +24,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj b/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj
index 1a0b4e3..ff11b7f 100644
--- a/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj
+++ b/src/DotJEM.Json.Index2.QueryParsers/DotJEM.Json.Index2.QueryParsers.csproj
@@ -20,7 +20,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj b/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj
index 2b975bb..5247129 100644
--- a/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj
+++ b/src/DotJEM.Json.Index2.Snapshots/DotJEM.Json.Index2.Snapshots.csproj
@@ -10,7 +10,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs b/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs
index cc7b1bc..d1ab3ab 100644
--- a/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs
+++ b/src/DotJEM.Json.Index2.Snapshots/IndexSnapshotHandler.cs
@@ -5,6 +5,7 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DotJEM.Json.Index2.IO;
+using DotJEM.Json.Index2.Leases;
using DotJEM.Json.Index2.Snapshots.Streams;
using Lucene.Net.Index;
using Lucene.Net.Store;
diff --git a/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj b/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj
index 63cb127..6bcb848 100644
--- a/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj
+++ b/src/DotJEM.Json.Index2.Test/DotJEM.Json.Index2.Test.csproj
@@ -22,7 +22,7 @@
-
+
diff --git a/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs b/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs
index 9251c5c..e118b60 100644
--- a/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs
+++ b/src/DotJEM.Json.Index2.Test/JsonIndexTest.cs
@@ -34,6 +34,7 @@ public async Task Create_AddsDocument()
//int count = searcher.Search(new MatchAllDocsQuery()).Count();
Assert.AreEqual(5, count);
}
+
[Test]
public async Task SayHello_ReturnsHello()
{
@@ -83,4 +84,26 @@ public async Task Search_Booleans()
int count = searcher.Search(new TermQuery(new Term("inStock", "true"))).Count();
Assert.AreEqual(3, count);
}
+ [Test]
+ public async Task FindBeforeCommit_AddsDocument()
+ {
+ IJsonIndex index = new JsonIndexBuilder("myIndex")
+ .UsingMemmoryStorage()
+ .WithAnalyzer(cfg => new StandardAnalyzer(cfg.Version))
+ .WithFieldResolver(new FieldResolver("uuid", "type"))
+ .Build();
+
+ IJsonIndexWriter writer = index.CreateWriter();
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+
+ IJsonIndexSearcher? searcher = index.CreateSearcher();
+ Assert.That(searcher.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(5));
+
+ writer.Create(JObject.FromObject(new { uuid = Guid.NewGuid(), type = "CAR" }));
+ Assert.That(searcher.Search(new TermQuery(new Term("type", "car"))).Count(), Is.EqualTo(6));
+ }
}
\ No newline at end of file
diff --git a/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj b/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj
index 153cda4..93efaa3 100644
--- a/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj
+++ b/src/DotJEM.Json.Index2/DotJEM.Json.Index2.csproj
@@ -44,9 +44,9 @@
-
-
-
+
+
+
diff --git a/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs b/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs
index af0b8df..1b6031f 100644
--- a/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs
+++ b/src/DotJEM.Json.Index2/IO/JsonIndexWriter.cs
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using DotJEM.Json.Index2.Documents;
+using DotJEM.Json.Index2.Leases;
using DotJEM.Json.Index2.Util;
using Lucene.Net.Documents;
using Lucene.Net.Index;
diff --git a/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs b/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs
index 60ad7d3..79cc124 100644
--- a/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs
+++ b/src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs
@@ -4,25 +4,24 @@
using System.Diagnostics;
using System.Linq;
using System.Threading;
+using System.Threading.Tasks;
using DotJEM.Json.Index2.Configuration;
+using DotJEM.Json.Index2.Leases;
using DotJEM.Json.Index2.Util;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
+using Lucene.Net.Search;
namespace DotJEM.Json.Index2.IO;
public interface IIndexWriterManager : IDisposable
{
event EventHandler OnClose;
+
ILease Lease();
void Close();
}
-public interface ILease : IDisposable
-{
- T Value { get; }
- bool IsExpired { get; }
-}
public class IndexWriterManager : Disposable, IIndexWriterManager
{
@@ -31,8 +30,9 @@ public class IndexWriterManager : Disposable, IIndexWriterManager
private readonly IJsonIndex index;
private volatile IndexWriter writer;
private readonly object writerPadLock = new();
- private readonly object leasesPadLock = new();
+ private readonly LeaseManager leaseManager = new();
+ //TODO: With leases, this should not be needed.
public event EventHandler OnClose;
private IndexWriter Writer
@@ -47,40 +47,13 @@ private IndexWriter Writer
if (writer != null)
return writer;
- try
- {
- return writer = Open(index);
- }
- catch (Exception e)
- {
- Debug.WriteLine("ACTIVE LEASES: " +leases.Count);
- throw;
- }
+ return writer = Open(index);
}
}
}
- private readonly List leases = new List();
-
- public ILease Lease()
- {
- //TODO: Optimizied collection for this.
- TimeLimitedIndexWriterLease lease = new(this, OnReturned);
- lock (leasesPadLock)
- {
- leases.Add(lease);
- }
- return lease;
- }
-
- private void OnReturned(TimeLimitedIndexWriterLease lease)
- {
- lock (leasesPadLock)
- {
- leases.Remove(lease);
- }
- }
+ public ILease Lease() => leaseManager.Create(Writer, TimeSpan.FromSeconds(3));
public IndexWriterManager(IJsonIndex index)
{
@@ -89,7 +62,6 @@ public IndexWriterManager(IJsonIndex index)
private static IndexWriter Open(IJsonIndex index)
{
- Debug.WriteLine("OPEN WRITER");
IndexWriterConfig config = new(index.Configuration.Version, index.Configuration.Analyzer);
config.RAMBufferSizeMB = DEFAULT_RAM_BUFFER_SIZE_MB;
config.OpenMode = OpenMode.CREATE_OR_APPEND;
@@ -103,28 +75,15 @@ public void Close()
if (writer == null)
return;
- lock (leasesPadLock)
+ lock (writerPadLock)
{
- TimeLimitedIndexWriterLease[] leasesCopy = leases.ToArray();
-
- Debug.WriteLine("ACTIVE LEASES: " + leasesCopy.Length);
- leases.Clear();
- foreach (TimeLimitedIndexWriterLease lease in leasesCopy)
- {
- if (!lease.IsExpired)
- lease.Wait();
- lease.Dispose();
- }
-
- lock (writerPadLock)
- {
- if (writer == null)
- return;
+ leaseManager.RecallAll();
+ if (writer == null)
+ return;
- writer.Dispose();
- writer = null;
- RaiseOnClose();
- }
+ writer.Dispose();
+ writer = null;
+ RaiseOnClose();
}
}
@@ -140,63 +99,4 @@ protected virtual void RaiseOnClose()
{
OnClose?.Invoke(this, EventArgs.Empty);
}
-
- private class TimeLimitedIndexWriterLease : Disposable, ILease
- {
- private readonly DateTime leaseTime = DateTime.Now;
- private readonly Action onReturned;
- private readonly IndexWriterManager manager;
- public AutoResetEvent Handle { get; } = new(false);
-
- public IndexWriter Value
- {
- get
- {
- if (IsDisposed)
- {
- throw new ObjectDisposedException("Index writer lease has been returned or is expired.");
- }
-
- if (IsExpired)
- {
- throw new LeaseExpiredException("Index writer lease has been returned or is expired.");
- }
- return manager.Writer;
- }
- }
-
- public bool IsExpired => (DateTime.Now - leaseTime > TimeSpan.FromSeconds(5)) || IsDisposed;
-
- public TimeLimitedIndexWriterLease(IndexWriterManager manager, Action onReturned)
- {
- this.onReturned = onReturned;
- this.manager = manager;
- }
-
- protected override void Dispose(bool disposing)
- {
- Handle.Set();
- onReturned(this);
- }
-
- public void Wait()
- {
- if (IsExpired)
- return;
-
- Handle.WaitOne(TimeSpan.FromSeconds(6) - (DateTime.Now - leaseTime));
- }
- }
-
}
-
-public class LeaseExpiredException : Exception
-{
- public LeaseExpiredException(string message) : base(message)
- {
- }
-
- public LeaseExpiredException(string message, Exception innerException) : base(message, innerException)
- {
- }
-}
\ No newline at end of file
diff --git a/src/DotJEM.Json.Index2/Leases/Lease.cs b/src/DotJEM.Json.Index2/Leases/Lease.cs
new file mode 100644
index 0000000..0df807c
--- /dev/null
+++ b/src/DotJEM.Json.Index2/Leases/Lease.cs
@@ -0,0 +1,18 @@
+using System;
+
+namespace DotJEM.Json.Index2.Leases;
+
+public interface ILease : IDisposable
+{
+ event EventHandler Terminated;
+
+ T Value { get; }
+ bool IsExpired { get; }
+ bool TryRenew();
+}
+
+
+public interface ILessor
+{
+ ILease Lease();
+}
diff --git a/src/DotJEM.Json.Index2/Leases/LeaseManager.cs b/src/DotJEM.Json.Index2/Leases/LeaseManager.cs
new file mode 100644
index 0000000..d89bc9e
--- /dev/null
+++ b/src/DotJEM.Json.Index2/Leases/LeaseManager.cs
@@ -0,0 +1,191 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using DotJEM.Json.Index2.Util;
+
+namespace DotJEM.Json.Index2.Leases;
+
+public interface ILeaseManager
+{
+ int Count { get; }
+ ILease Create(T value);
+ ILease Create(T value, TimeSpan limit);
+ void RecallAll();
+}
+
+public class LeaseManager : ILeaseManager
+{
+ //TODO: Optimizied collection for this.
+ private readonly List> leases = new();
+ private readonly object leasesPadLock = new();
+
+ public int Count => leases.Count;
+
+ public ILease Create(T value)
+ {
+ return Add(new Lease(value, OnReturned));
+ }
+
+ public ILease Create(T value, TimeSpan limit)
+ {
+ return Add(new TimeLimitedLease(value, OnReturned, limit));
+ }
+
+ public void RecallAll()
+ {
+ lock (leasesPadLock)
+ {
+ IRecallableLease[] copy = leases.ToArray();
+ leases.Clear();
+
+ foreach (IRecallableLease lease in copy)
+ lease.Terminate();
+ }
+ }
+
+ private ILease Add(IRecallableLease lease)
+ {
+ lock (leasesPadLock)
+ {
+ leases.Add(lease);
+ }
+ return lease;
+ }
+
+ private void OnReturned(IRecallableLease obj)
+ {
+ lock (leasesPadLock)
+ {
+ leases.Remove(obj);
+ }
+ }
+
+ private interface IRecallableLease : ILease
+ {
+ void Terminate();
+ }
+
+ private class Lease : Disposable, IRecallableLease
+ {
+ public event EventHandler Terminated;
+
+ private readonly T value;
+ private readonly Action> onReturned;
+
+ public bool IsExpired => IsDisposed;
+
+
+ public T Value
+ {
+ get
+ {
+ if (IsExpired)
+ {
+ throw new LeaseExpiredException("Lease is expired as it was returned.");
+ }
+ return value;
+ }
+ }
+
+ public Lease(T value, Action> onReturned)
+ {
+ this.value = value;
+ this.onReturned = onReturned;
+ }
+
+ public bool TryRenew()
+ {
+ return false;
+ }
+
+ public void Terminate()
+ {
+ Terminated?.Invoke(this, EventArgs.Empty);
+ Dispose();
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ onReturned(this);
+ base.Dispose(disposing);
+ }
+ }
+
+ private class TimeLimitedLease : Disposable, IRecallableLease
+ {
+ public event EventHandler Terminated;
+
+ private readonly T value;
+ private readonly Action> onReturned;
+ private readonly TimeSpan timeLimit;
+ private readonly DateTime leaseTime = DateTime.Now;
+ private readonly AutoResetEvent handle = new(false);
+
+ public bool IsExpired => (DateTime.Now - leaseTime > timeLimit) || IsDisposed;
+
+ public T Value
+ {
+ get
+ {
+ if (IsExpired)
+ {
+ throw new LeaseExpiredException($"Lease is expired either because the time limit '{timeLimit}' has exceeded or the lease was returned.");
+ }
+ return value;
+ }
+ }
+
+ public TimeLimitedLease(T value, Action> onReturned, TimeSpan timeLimit)
+ {
+ this.value = value;
+ this.onReturned = onReturned;
+ this.timeLimit = timeLimit;
+ }
+ public bool TryRenew()
+ {
+ return false;
+ }
+
+ public void Terminate()
+ {
+ Terminated?.Invoke(this, EventArgs.Empty);
+ Wait();
+ Dispose();
+ }
+
+ public void Wait()
+ {
+ if (IsExpired)
+ return;
+
+ handle.WaitOne(TimeSpan.FromSeconds(6) - (DateTime.Now - leaseTime));
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ onReturned(this);
+ base.Dispose(disposing);
+ }
+ }
+}
+public class LeaseExpiredException : Exception
+{
+ public LeaseExpiredException(string message) : base(message)
+ {
+ }
+
+ public LeaseExpiredException(string message, Exception innerException) : base(message, innerException)
+ {
+ }
+}
+
+public class LeaseTerminatedException : Exception
+{
+ public LeaseTerminatedException(string message) : base(message)
+ {
+ }
+
+ public LeaseTerminatedException(string message, Exception innerException) : base(message, innerException)
+ {
+ }
+}
\ No newline at end of file
diff --git a/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs b/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs
index e79b925..abd446a 100644
--- a/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs
+++ b/src/DotJEM.Json.Index2/Storage/IIndexStorageProvider.cs
@@ -1,4 +1,7 @@
-using Lucene.Net.Store;
+using System;
+using System.IO;
+using Lucene.Net.Store;
+using Directory = Lucene.Net.Store.Directory;
namespace DotJEM.Json.Index2.Storage;
@@ -32,5 +35,12 @@ public SimpleFsIndexStorageProvider(string path)
public void Delete()
{
+ //TODO: For now. But maybe there is cases where this actually makes sense to always have.
+ DirectoryInfo dir = new DirectoryInfo(path);
+ foreach (FileInfo file in dir.EnumerateFiles())
+ file.Delete();
+
+ foreach (DirectoryInfo directory in dir.EnumerateDirectories())
+ directory.Delete(true);
}
}
\ No newline at end of file
diff --git a/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs b/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs
index 9402776..cf2738a 100644
--- a/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs
+++ b/src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs
@@ -1,6 +1,7 @@
using System;
using System.Text.RegularExpressions;
using DotJEM.Json.Index2.IO;
+using DotJEM.Json.Index2.Leases;
using DotJEM.Json.Index2.Searching;
using Lucene.Net.Index;
using Lucene.Net.Store;
@@ -23,6 +24,7 @@ public class JsonIndexStorageManager: IJsonIndexStorageManager
private readonly IIndexStorageProvider provider;
private readonly object padlock = new ();
private volatile Directory directory;
+ private readonly LeaseManager leaseManager = new();
private readonly Lazy writerManager;
private readonly Lazy searcherManager;
@@ -40,7 +42,6 @@ public Directory Directory
{
if (directory != null)
return directory;
-
return directory = provider.Get();
}
}
@@ -49,8 +50,8 @@ public Directory Directory
public JsonIndexStorageManager(IJsonIndex index, IIndexStorageProvider provider)
{
this.provider = provider;
- this.writerManager = new Lazy(()=> new IndexWriterManager(index));
- this.searcherManager = new Lazy(()=> new IndexSearcherManager(WriterManager, index.Configuration.Serializer));
+ this.writerManager = new(()=> new IndexWriterManager(index));
+ this.searcherManager = new(()=> new IndexSearcherManager(WriterManager, index.Configuration.Serializer));
}
public void Unlock()
@@ -80,8 +81,6 @@ public void Delete()
foreach (string file in directory.ListAll())
directory.DeleteFile(file);
provider.Delete();
-
-
}
}
}
\ No newline at end of file
diff --git a/src/Stress/StressTester/Program.cs b/src/Stress/StressTester/Program.cs
index ac4cf1b..06f1947 100644
--- a/src/Stress/StressTester/Program.cs
+++ b/src/Stress/StressTester/Program.cs
@@ -11,6 +11,7 @@
using DotJEM.Json.Index2.Management.Info;
using DotJEM.Json.Index2.Management.Snapshots;
using DotJEM.Json.Index2.Management.Snapshots.Zip;
+using DotJEM.Json.Index2.Management.Source;
using DotJEM.Json.Index2.Management.Tracking;
using DotJEM.Json.Index2.Management.Writer;
using DotJEM.Json.Index2.Searching;
@@ -29,8 +30,9 @@
//TraceSource trace;
+IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=SSN3DB;Integrated Security=True");
//IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=nsw;Integrated Security=True");
-IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True");
+//IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True");
storage.Configure.MapField(JsonField.Id, "id");
storage.Configure.MapField(JsonField.ContentType, "contentType");
storage.Configure.MapField(JsonField.Version, "$version");
@@ -38,19 +40,19 @@
storage.Configure.MapField(JsonField.Updated, "$updated");
storage.Configure.MapField(JsonField.SchemaVersion, "$schemaVersion");
-StressDataGenerator generator = new StressDataGenerator(
- storage.Area("Settings"),
- storage.Area("Queue"),
- storage.Area("Recipes"),
- storage.Area("Animals"),
- storage.Area("Games"),
- storage.Area("Players"),
- storage.Area("Planets"),
- storage.Area("Universe"),
- storage.Area("Trashcan")
-);
-Task genTask = generator.StartAsync();
-await Task.Delay(2000);
+//StressDataGenerator generator = new StressDataGenerator(
+// storage.Area("Settings"),
+// storage.Area("Queue"),
+// storage.Area("Recipes"),
+// storage.Area("Animals"),
+// storage.Area("Games"),
+// storage.Area("Players"),
+// storage.Area("Planets"),
+// storage.Area("Universe"),
+// storage.Area("Trashcan")
+//);
+//Task genTask = generator.StartAsync();
+//await Task.Delay(2000);
if (Directory.Exists(@".\app_data\index"))
Directory.Delete(@".\app_data\index", true);
@@ -67,12 +69,26 @@
string[] areas = new[] { "content", "settings", "diagnostic", "emsaqueue", "statistic" };
IWebTaskScheduler scheduler = new WebTaskScheduler();
+IJsonDocumentSource jsonStorageDocumentSource = new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler, areas));
IJsonIndexManager jsonIndexManager = new JsonIndexManager(
- new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler,areas)),
+ jsonStorageDocumentSource,
new JsonIndexSnapshotManager(index, new ZipSnapshotStrategy(".\\app_data\\snapshots"), scheduler, "30m"),
index
);
+
+DateTime start = DateTime.Now;
+TimeSpan completedAfter = TimeSpan.Zero;
+jsonStorageDocumentSource.Initialized
+ .Where(b => b)
+ .FirstAsync(b =>
+ {
+ completedAfter = DateTime.Now - start;
+ Console.WriteLine($"Done after: {completedAfter:g}");
+ return b;
+ });
+
+
Task run = Task.WhenAll(
jsonIndexManager.InfoStream.ForEachAsync(Reporter.CaptureInfo),
jsonIndexManager.RunAsync()
@@ -123,10 +139,10 @@ public static class Reporter
{
private static ITrackerState lastState;
private static IInfoStreamEvent lastEvent;
- private static DateTime lastReport = DateTime.Now;
+ private static DateTime lastReport = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));
private static readonly Queue messages = new Queue();
-
+ private static long eventCounter = 0;
public static void CaptureInfo(IInfoStreamEvent evt)
{
lock (messages)
@@ -147,11 +163,11 @@ public static void CaptureInfo(IInfoStreamEvent evt)
break;
default:
-
lastEvent = evt;
break;
}
- Report();
+ if(eventCounter++ % 10000 == 0)
+ Console.Write('.');
}
private static string CleanLine = new string(' ', Console.BufferWidth);
@@ -160,7 +176,7 @@ public static void CaptureInfo(IInfoStreamEvent evt)
public static void Report(bool force = false)
{
- if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(30))
+ if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(5))
return;
lastReport = DateTime.Now;
int lines = nl.Matches(buffer.ToString()).Count;
@@ -176,15 +192,11 @@ public static void Report(bool force = false)
{
msgs = messages.ToArray();
}
-
+ //Console.WriteLine(lastEvent.Message);
+ buffer.AppendLine(lastState?.ToString());
foreach (string message in msgs)
- {
buffer.AppendLine(message);
- }
-
buffer.AppendLine();
- //Console.WriteLine(lastEvent.Message);
- buffer.AppendLine(lastState.ToString());
Console.WriteLine(buffer);
}
}
diff --git a/src/Stress/StressTester/StressTester.csproj b/src/Stress/StressTester/StressTester.csproj
index aa7b510..c3ae235 100644
--- a/src/Stress/StressTester/StressTester.csproj
+++ b/src/Stress/StressTester/StressTester.csproj
@@ -12,9 +12,9 @@
-
-
-
+
+
+