Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DocumentsWriterThreadPool upgrade to 4.8.1 (fixes TestRollbackAndCommitWithThreads() test) #335

Merged
merged 2 commits into from
Sep 7, 2020
Merged

DocumentsWriterThreadPool upgrade to 4.8.1 (fixes TestRollbackAndCommitWithThreads() test) #335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1eadb1a
LUCENE-5644: switch to simpler LIFO thread to ThreadState allocator d…
vvdb Jun 13, 2017
2a7b0f2
Fixed merge conflicts/other issues for patch LUCENE-5644 (see #208)
NightOwl888 Aug 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 0 additions & 102 deletions src/Lucene.Net.TestFramework/Index/RandomDocumentsWriterPerThreadPool.cs
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,4 @@
<InternalsVisibleTo Include="Lucene.Net.Tests.TestFramework.DependencyInjection" />
</ItemGroup>

</Project>
</Project>
28 changes: 2 additions & 26 deletions src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1606,32 +1606,8 @@ public static IndexWriterConfig NewIndexWriterConfig(LuceneTestCase luceneTestCa
{
int maxNumThreadStates = Rarely(random) ? TestUtil.NextInt32(random, 5, 20) : TestUtil.NextInt32(random, 1, 4); // reasonable value - crazy value

if (Rarely(random))
{
//// Retrieve the package-private setIndexerThreadPool
//// method:
////MethodInfo setIndexerThreadPoolMethod = typeof(IndexWriterConfig).GetMethod("SetIndexerThreadPool", new Type[] { typeof(DocumentsWriterPerThreadPool) });
//MethodInfo setIndexerThreadPoolMethod = typeof(IndexWriterConfig).GetMethod(
// "SetIndexerThreadPool",
// BindingFlags.NonPublic | BindingFlags.Instance,
// null,
// new Type[] { typeof(DocumentsWriterPerThreadPool) },
// null);
////setIndexerThreadPoolMethod.setAccessible(true);
//Type clazz = typeof(RandomDocumentsWriterPerThreadPool);
//ConstructorInfo ctor = clazz.GetConstructor(new[] { typeof(int), typeof(Random) });
////ctor.Accessible = true;
//// random thread pool
//setIndexerThreadPoolMethod.Invoke(c, new[] { ctor.Invoke(new object[] { maxNumThreadStates, r }) });

// LUCENENET specific: Since we are using InternalsVisibleTo, there is no need for Reflection
c.SetIndexerThreadPool(new RandomDocumentsWriterPerThreadPool(maxNumThreadStates, random));
}
else
{
// random thread pool
c.SetMaxThreadStates(maxNumThreadStates);
}
// LUCENENET specific - Removed RandomDocumentsWriterPerThreadPool, as was done in Lucene 4.8.1 (see #208)
c.SetMaxThreadStates(maxNumThreadStates);
}
#if !FEATURE_INSTANCE_TESTDATA_INITIALIZATION
c.SetMergePolicy(NewMergePolicy(random));
Expand Down
8 changes: 4 additions & 4 deletions src/Lucene.Net.Tests/Index/TestFlushByRamOrCountsPolicy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, b

IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Expand Down Expand Up @@ -153,7 +153,7 @@ public virtual void TestFlushDocCount()
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);

int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Expand Down Expand Up @@ -206,7 +206,7 @@ public virtual void TestRandom()
iwc.SetFlushPolicy(flushPolicy);

int numDWPT = 1 + Random.Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);

IndexWriter writer = new IndexWriter(dir, iwc);
Expand Down Expand Up @@ -277,7 +277,7 @@ public virtual void TestStallControl()
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);

DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
Expand Down
2 changes: 1 addition & 1 deletion src/Lucene.Net.Tests/Index/TestIndexWriterConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public virtual void TestDefaults()
Assert.IsNull(conf.MergedSegmentWarmer);
Assert.AreEqual(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.ReaderTermsIndexDivisor);
Assert.AreEqual(typeof(TieredMergePolicy), conf.MergePolicy.GetType());
Assert.AreEqual(typeof(ThreadAffinityDocumentsWriterThreadPool), conf.IndexerThreadPool.GetType());
Assert.AreEqual(typeof(DocumentsWriterPerThreadPool), conf.IndexerThreadPool.GetType());
Assert.AreEqual(typeof(FlushByRamOrCountsPolicy), conf.FlushPolicy.GetType());
Assert.AreEqual(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.RAMPerThreadHardLimitMB);
Assert.AreEqual(Codec.Default, conf.Codec);
Expand Down
2 changes: 1 addition & 1 deletion src/Lucene.Net.Tests/Index/TestStressIndexing2.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ public virtual DocsAndWriter IndexRandomIWReader(int nThreads, int iterations, i
public virtual IDictionary<string, Document> IndexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates, bool doReaderPooling)
{
IDictionary<string, Document> docs = new Dictionary<string, Document>();
IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetRAMBufferSizeMB(0.1).SetMaxBufferedDocs(maxBufferedDocs).SetIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(maxThreadStates)).SetReaderPooling(doReaderPooling).SetMergePolicy(NewLogMergePolicy()), new YieldTestPoint(this));
IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetRAMBufferSizeMB(0.1).SetMaxBufferedDocs(maxBufferedDocs).SetIndexerThreadPool(new DocumentsWriterPerThreadPool(maxThreadStates)).SetReaderPooling(doReaderPooling).SetMergePolicy(NewLogMergePolicy()), new YieldTestPoint(this));
LogMergePolicy lmp = (LogMergePolicy)w.Config.MergePolicy;
lmp.NoCFSRatio = 0.0;
lmp.MergeFactor = mergeFactor;
Expand Down
10 changes: 5 additions & 5 deletions src/Lucene.Net/Index/DocumentsWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,7 @@ internal bool UpdateDocuments(IEnumerable<IEnumerable<IIndexableField>> docs, An
}
finally
{
perThread.Unlock();
perThreadPool.Release(perThread);
}

return PostUpdate(flushingDWPT, hasEvents);
Expand Down Expand Up @@ -572,7 +572,7 @@ internal bool UpdateDocument(IEnumerable<IIndexableField> doc, Analyzer analyzer
}
finally
{
perThread.Unlock();
perThreadPool.Release(perThread);
}

return PostUpdate(flushingDWPT, hasEvents);
Expand Down Expand Up @@ -809,7 +809,7 @@ private void PutEvent(IEvent @event)
internal sealed class ApplyDeletesEvent : IEvent
{
internal static readonly IEvent INSTANCE = new ApplyDeletesEvent();
private int instCount = 0;
private static int instCount = 0; // LUCENENET: Made static, otherwise this makes no sense at all

internal ApplyDeletesEvent()
{
Expand All @@ -826,7 +826,7 @@ public void Process(IndexWriter writer, bool triggerMerge, bool forcePurge)
internal sealed class MergePendingEvent : IEvent
{
internal static readonly IEvent INSTANCE = new MergePendingEvent();
private int instCount = 0;
private static int instCount = 0; // LUCENENET: Made static, otherwise this makes no sense at all

internal MergePendingEvent()
{
Expand All @@ -843,7 +843,7 @@ public void Process(IndexWriter writer, bool triggerMerge, bool forcePurge)
internal sealed class ForcedPurgeEvent : IEvent
{
internal static readonly IEvent INSTANCE = new ForcedPurgeEvent();
private int instCount = 0;
private static int instCount = 0; // LUCENENET: Made static, otherwise this makes no sense at all

internal ForcedPurgeEvent()
{
Expand Down
4 changes: 2 additions & 2 deletions src/Lucene.Net/Index/DocumentsWriterFlushControl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,8 +613,8 @@ internal ThreadState ObtainAndLock()
{
if (!success) // make sure we unlock if this fails
{
perThread.Unlock();
}
perThreadPool.Release(perThread);
}
}
}

Expand Down
Loading