Skip to content

Commit

Permalink
Merge branch 'issues/GH-6' of https://github.com/dotJEM/json-index2 i…
Browse files Browse the repository at this point in the history 
…nto issues/GH-6
  • Loading branch information
jmd committed Nov 7, 2024
2 parents 898da46 + 9d2cf5e commit 1e59c7f
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 62 deletions.
44 changes: 8 additions & 36 deletions src/DotJEM.Json.Index2/IO/JsonIndexWriterManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ public class IndexWriterManager : Disposable, IIndexWriterManager
private readonly IJsonIndex index;
private volatile IndexWriter writer;
private readonly object writerPadLock = new();
private readonly object leasesPadLock = new();
private readonly LeaseManager<IndexWriter> leaseManager = new();

//TODO: With leases, this should not be needed.
Expand All @@ -48,24 +47,13 @@ private IndexWriter Writer
if (writer != null)
return writer;

try
{
return writer = Open(index);
}
catch (Exception e)
{
Debug.WriteLine("ACTIVE LEASES: " + leaseManager.Count);
throw;
}
return writer = Open(index);
}
}
}


public ILease<IndexWriter> Lease()
{
return leaseManager.Create(Writer, TimeSpan.FromSeconds(3));
}
public ILease<IndexWriter> Lease() => leaseManager.Create(Writer, TimeSpan.FromSeconds(3));

public IndexWriterManager(IJsonIndex index)
{
Expand All @@ -74,7 +62,6 @@ public IndexWriterManager(IJsonIndex index)

private static IndexWriter Open(IJsonIndex index)
{
Debug.WriteLine("OPEN WRITER");
IndexWriterConfig config = new(index.Configuration.Version, index.Configuration.Analyzer);
config.RAMBufferSizeMB = DEFAULT_RAM_BUFFER_SIZE_MB;
config.OpenMode = OpenMode.CREATE_OR_APPEND;
Expand All @@ -88,30 +75,15 @@ public void Close()
if (writer == null)
return;

lock (leasesPadLock)
lock (writerPadLock)
{
//TimeLimitedLease<IndexWriter>[] leasesCopy = leases.ToArray();

leaseManager.RecallAll();
if (writer == null)
return;

//Debug.WriteLine("ACTIVE LEASES: " + leasesCopy.Length);
//leases.Clear();
//foreach (TimeLimitedLease<IndexWriter> lease in leasesCopy)
//{
// if (!lease.IsExpired)
// lease.Wait();
// lease.Dispose();
//}

lock (writerPadLock)
{
if (writer == null)
return;

writer.Dispose();
writer = null;
RaiseOnClose();
}
writer.Dispose();
writer = null;
RaiseOnClose();
}
}

Expand Down
2 changes: 2 additions & 0 deletions src/DotJEM.Json.Index2/Storage/IJsonIndexStorageManager.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Text.RegularExpressions;
using DotJEM.Json.Index2.IO;
using DotJEM.Json.Index2.Leases;
using DotJEM.Json.Index2.Searching;
using Lucene.Net.Index;
using Lucene.Net.Store;
Expand All @@ -23,6 +24,7 @@ public class JsonIndexStorageManager: IJsonIndexStorageManager
private readonly IIndexStorageProvider provider;
private readonly object padlock = new ();
private volatile Directory directory;
private readonly LeaseManager<Directory> leaseManager = new();

private readonly Lazy<IIndexWriterManager> writerManager;
private readonly Lazy<IIndexSearcherManager> searcherManager;
Expand Down
64 changes: 38 additions & 26 deletions src/Stress/StressTester/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using DotJEM.Json.Index2.Management.Info;
using DotJEM.Json.Index2.Management.Snapshots;
using DotJEM.Json.Index2.Management.Snapshots.Zip;
using DotJEM.Json.Index2.Management.Source;
using DotJEM.Json.Index2.Management.Tracking;
using DotJEM.Json.Index2.Management.Writer;
using DotJEM.Json.Index2.Searching;
Expand All @@ -29,28 +30,29 @@

//TraceSource trace;

IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=SSN3DB;Integrated Security=True");
//IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=nsw;Integrated Security=True");
IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True");
//IStorageContext storage = new SqlServerStorageContext("Data Source=.\\DEV;Initial Catalog=STRESS;Integrated Security=True");
storage.Configure.MapField(JsonField.Id, "id");
storage.Configure.MapField(JsonField.ContentType, "contentType");
storage.Configure.MapField(JsonField.Version, "$version");
storage.Configure.MapField(JsonField.Created, "$created");
storage.Configure.MapField(JsonField.Updated, "$updated");
storage.Configure.MapField(JsonField.SchemaVersion, "$schemaVersion");

StressDataGenerator generator = new StressDataGenerator(
storage.Area("Settings"),
storage.Area("Queue"),
storage.Area("Recipes"),
storage.Area("Animals"),
storage.Area("Games"),
storage.Area("Players"),
storage.Area("Planets"),
storage.Area("Universe"),
storage.Area("Trashcan")
);
Task genTask = generator.StartAsync();
await Task.Delay(2000);
//StressDataGenerator generator = new StressDataGenerator(
// storage.Area("Settings"),
// storage.Area("Queue"),
// storage.Area("Recipes"),
// storage.Area("Animals"),
// storage.Area("Games"),
// storage.Area("Players"),
// storage.Area("Planets"),
// storage.Area("Universe"),
// storage.Area("Trashcan")
//);
//Task genTask = generator.StartAsync();
//await Task.Delay(2000);

if (Directory.Exists(@".\app_data\index"))
Directory.Delete(@".\app_data\index", true);
Expand All @@ -67,12 +69,26 @@
string[] areas = new[] { "content", "settings", "diagnostic", "emsaqueue", "statistic" };

IWebTaskScheduler scheduler = new WebTaskScheduler();
IJsonDocumentSource jsonStorageDocumentSource = new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler, areas));
IJsonIndexManager jsonIndexManager = new JsonIndexManager(
new JsonStorageDocumentSource(new JsonStorageAreaObserverFactory(storage, scheduler,areas)),
jsonStorageDocumentSource,
new JsonIndexSnapshotManager(index, new ZipSnapshotStrategy(".\\app_data\\snapshots"), scheduler, "30m"),
index
);


DateTime start = DateTime.Now;
TimeSpan completedAfter = TimeSpan.Zero;
jsonStorageDocumentSource.Initialized
.Where(b => b)
.FirstAsync(b =>
{
completedAfter = DateTime.Now - start;
Console.WriteLine($"Done after: {completedAfter:g}");
return b;
});


Task run = Task.WhenAll(
jsonIndexManager.InfoStream.ForEachAsync(Reporter.CaptureInfo),
jsonIndexManager.RunAsync()
Expand Down Expand Up @@ -123,10 +139,10 @@ public static class Reporter
{
private static ITrackerState lastState;
private static IInfoStreamEvent lastEvent;
private static DateTime lastReport = DateTime.Now;
private static DateTime lastReport = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));

private static readonly Queue<string> messages = new Queue<string>();

private static long eventCounter = 0;
public static void CaptureInfo(IInfoStreamEvent evt)
{
lock (messages)
Expand All @@ -147,11 +163,11 @@ public static void CaptureInfo(IInfoStreamEvent evt)
break;

default:

lastEvent = evt;
break;
}
Report();
if(eventCounter++ % 10000 == 0)
Console.Write('.');
}

private static string CleanLine = new string(' ', Console.BufferWidth);
Expand All @@ -160,7 +176,7 @@ public static void CaptureInfo(IInfoStreamEvent evt)

public static void Report(bool force = false)
{
if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(30))
if(!force && DateTime.Now - lastReport < TimeSpan.FromSeconds(5))
return;
lastReport = DateTime.Now;
int lines = nl.Matches(buffer.ToString()).Count;
Expand All @@ -176,15 +192,11 @@ public static void Report(bool force = false)
{
msgs = messages.ToArray();
}

//Console.WriteLine(lastEvent.Message);
buffer.AppendLine(lastState?.ToString());
foreach (string message in msgs)
{
buffer.AppendLine(message);
}

buffer.AppendLine();
//Console.WriteLine(lastEvent.Message);
buffer.AppendLine(lastState.ToString());
Console.WriteLine(buffer);
}
}

0 comments on commit 1e59c7f

Please sign in to comment.