Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lucene index amend improvements #14513

Merged
merged 16 commits into from
Oct 27, 2023
1 change: 1 addition & 0 deletions src/DynamoCore/Search/NodeSearchModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
}
}
}
luceneSearchUtility.dirReader.Dispose();
return candidates;
}
return null;
Expand Down
36 changes: 31 additions & 5 deletions src/DynamoCore/Utilities/LuceneSearchUtility.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.IO;
using System.Linq;
using Dynamo.Configuration;
using Dynamo.Core;
using Dynamo.Events;
using Dynamo.Logging;
using Dynamo.Models;
Expand Down Expand Up @@ -62,6 +61,11 @@ internal class LuceneSearchUtility
/// </summary>
internal LuceneStartConfig startConfig;

/// <summary>
/// Index open mode for Lucene index writer
/// </summary>
internal OpenMode openMode { get; set; }

/// <summary>
/// Default start config for Lucene, it will use RAM storage type and empty directory
/// </summary>
Expand Down Expand Up @@ -95,6 +99,7 @@ public enum LuceneStorage
/// <summary>
/// Constructor for LuceneSearchUtility, it will use the storage type passed as parameter
/// </summary>
/// <param name="model"></param>
/// <param name="config"></param>
internal LuceneSearchUtility(DynamoModel model, LuceneStartConfig config)
{
Expand Down Expand Up @@ -132,12 +137,13 @@ internal void InitializeLuceneConfig()
/// <summary>
/// Create index writer for followup doc indexing
/// </summary>
internal void CreateLuceneIndexWriter()
/// <param name="mode"></param>
internal void CreateLuceneIndexWriter(OpenMode mode = OpenMode.CREATE)
{
// Create an index writer
IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer)
{
OpenMode = OpenMode.CREATE
OpenMode = mode
};
try
{
Expand All @@ -147,12 +153,12 @@ internal void CreateLuceneIndexWriter()
{

DisposeWriter();
(ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET LockObtainFailedException {ex}");
dynamoModel.Logger.LogError($"LuceneNET LockObtainFailedException {ex}");
Copy link
Contributor Author

@QilongTang QilongTang Oct 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to revert this because I found in edge case if the index is ever locked (although would rarely happen after all the improvements in this PR), the previous code will make Dynamo hang because of threading issues


}
catch (Exception ex)
{
(ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET Exception {ex}");
dynamoModel.Logger.LogError($"LuceneNET Exception {ex}");
}
}

Expand Down Expand Up @@ -384,12 +390,24 @@ internal Analyzer CreateAnalyzerByLanguage(string language)
}
}

/// <summary>
/// Dispose Lucene index write objects and reuse other objects
/// </summary>
internal void DisposeWriter()
{
writer?.Dispose();
writer = null;
}

/// <summary>
/// Dispose Lucene index read objects and reuse other objects
/// </summary>
internal void DisposeReader()
{
dirReader?.Dispose();
dirReader = null;
}

/// <summary>
/// Dispose all the Lucene objects
/// </summary>
Expand Down Expand Up @@ -418,6 +436,14 @@ internal void CommitWriterChanges()
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
{
if (addedFields == null) return;
// During DynamoModel initialization, the index writer should still be valid here
// If the index writer is null and index not locked, it means the index writer has been disposed, most likely due to DynamoView already launched
// If the index writer is null and index locked, it means user is in a secondary Dynamo session
// Then create a new index writer to amend the index should be safe
if (writer == null && !IndexWriter.IsLocked(this.indexDir))
reddyashish marked this conversation as resolved.
Show resolved Hide resolved
{
CreateLuceneIndexWriter(OpenMode.CREATE_OR_APPEND);
}

SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
Expand Down
3 changes: 2 additions & 1 deletion src/DynamoCoreWpf/Controls/StartPage.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Collections.Specialized;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Input;
Expand Down Expand Up @@ -366,7 +367,7 @@ private void RefreshFileList(ObservableCollection<StartPageListItem> files,
IEnumerable<string> filePaths)
{
files.Clear();
foreach (var filePath in filePaths)
foreach (var filePath in filePaths.Where(x => x != null))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is throwing a bunch of exceptions so filtering out the nulls from the list

{
try
{
Expand Down
22 changes: 11 additions & 11 deletions src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs
Original file line number Diff line number Diff line change
@@ -1,14 +1,3 @@
using Dynamo.Configuration;
using Dynamo.Engine.CodeGeneration;
using Dynamo.Graph;
using Dynamo.Graph.Nodes;
using Dynamo.Graph.Nodes.CustomNodes;
using Dynamo.Graph.Workspaces;
using Dynamo.Logging;
using Dynamo.Models;
using Dynamo.Selection;
using Dynamo.Wpf.ViewModels.Core;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
Expand All @@ -20,6 +9,17 @@
using System.Windows;
using System.Windows.Controls.Primitives;
using System.Windows.Media;
using Dynamo.Configuration;
using Dynamo.Engine.CodeGeneration;
using Dynamo.Graph;
using Dynamo.Graph.Nodes;
using Dynamo.Graph.Nodes.CustomNodes;
using Dynamo.Graph.Workspaces;
using Dynamo.Logging;
using Dynamo.Models;
using Dynamo.Selection;
using Dynamo.Wpf.ViewModels.Core;
using Newtonsoft.Json;
using Point = System.Windows.Point;
using Size = System.Windows.Size;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1577,7 +1577,7 @@ internal void Close()
InitialResultsLoaded = false; // reset the loading screen settings
RequestShowFileDialog -= OnRequestShowFileDialog;
nonHostFilter.ForEach(f => f.PropertyChanged -= filter_PropertyChanged);
LuceneUtility.DisposeAll();
LuceneUtility.DisposeReader();
}
}
}
Loading