diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs
index d40097b681b..982e1d0c508 100644
--- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs
+++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs
@@ -21,6 +21,16 @@ internal class LuceneSearchUtility
internal Lucene.Net.Store.Directory indexDir;
internal IndexWriter writer;
internal string directory;
+ internal LuceneStorage currentStorageType;
+
+ public enum LuceneStorage
+ {
+ //Lucene Storage will be located in RAM and all the info indexed will be lost when Dynamo app is closed
+ RAM,
+
+ //Lucene Storage will be located in the local File System and the files will remain in ...AppData\Roaming\Dynamo\Dynamo Core\2.19\Index folder
+ FILE_SYSTEM
+ }
// Used for creating the StandardAnalyzer
internal Analyzer Analyzer;
@@ -36,23 +46,34 @@ internal LuceneSearchUtility(DynamoModel model)
///
/// Initialize Lucene config file writer.
///
- internal void InitializeLuceneConfig(string dirName)
+ internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM)
{
addedFields = new List();
- DirectoryInfo webBrowserUserDataFolder;
+ DirectoryInfo luceneUserDataFolder;
var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory);
- webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null;
+ luceneUserDataFolder = userDataDir.Exists ? userDataDir : null;
directory = dirName;
- string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName);
- indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath);
+ string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName);
+
+ currentStorageType = storageType;
+
+ if (storageType == LuceneStorage.RAM)
+ {
+ indexDir = new RAMDirectory();
+ }
+ else
+ {
+ indexDir = FSDirectory.Open(indexPath);
+ }
+
// Create an analyzer to process the text
Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion);
- // Initialize Lucene index writer, unless in test mode.
- if (!DynamoModel.IsTestMode)
+ // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info.
+ if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
{
// Create an index writer
IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer)
@@ -77,7 +98,7 @@ internal void InitializeLuceneConfig(string dirName)
///
internal Document InitializeIndexDocumentForNodes()
{
- if (DynamoModel.IsTestMode) return null;
+ if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null;
var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES);
var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES);
@@ -153,7 +174,7 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo
((StringField)doc.GetField(field)).SetStringValue(value);
}
- if (isLast && indexedFields.Any())
+ if (isLast && indexedFields != null && indexedFields.Any())
{
List diff = indexedFields.Except(addedFields).ToList();
foreach (var d in diff)
@@ -248,7 +269,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm)
internal void DisposeWriter()
{
//We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location
- if (!DynamoModel.IsTestMode)
+ if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
{
writer?.Dispose();
writer = null;
@@ -257,7 +278,7 @@ internal void DisposeWriter()
internal void CommitWriterChanges()
{
- if (!DynamoModel.IsTestMode)
+ if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
{
//Commit the info indexed
writer?.Commit();
diff --git a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
index 66748db5eb3..68677882d28 100644
--- a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
+++ b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
@@ -18,6 +18,9 @@
using Dynamo.Utilities;
using Dynamo.Wpf.ViewModels;
using Greg;
+using Lucene.Net.Documents;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
using Newtonsoft.Json;
using ProtoCore.AST.AssociativeAST;
using ProtoCore.Mirror;
@@ -39,6 +42,9 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel
private bool displayLowConfidence;
private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete";
+ // Lucene search utility to perform indexing operations just for NodeAutocomplete.
+ internal LuceneSearchUtility LuceneSearchUtilityNodeAutocomplete { get; set; }
+
///
/// The Node AutoComplete ML service version, this could be empty if user has not used ML way
///
@@ -602,6 +608,61 @@ private NodeSearchElementViewModel GetViewModelForNodeSearchElement(NodeSearchEl
return null;
}
+
+ ///
+ /// Performs a search using the given string as query and subset, if provided.
+ ///
+ /// Returns a list with a maximum MaxNumSearchResults elements.
+ /// The search query
+ /// Temporary flag that will be used for searching using Lucene.NET
+ internal IEnumerable SearchNodeAutocomplete(string search, bool useLucene)
+ {
+ if (useLucene)
+ {
+ //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
+ LuceneSearchUtilityNodeAutocomplete.dirReader = LuceneSearchUtilityNodeAutocomplete.writer?.GetReader(applyAllDeletes: true);
+ if (LuceneSearchUtilityNodeAutocomplete.dirReader == null) return null;
+
+ LuceneSearchUtilityNodeAutocomplete.Searcher = new IndexSearcher(LuceneSearchUtilityNodeAutocomplete.dirReader);
+
+ string searchTerm = search.Trim();
+ var candidates = new List();
+ var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtilityNodeAutocomplete.Analyzer)
+ {
+ AllowLeadingWildcard = true,
+ DefaultOperator = LuceneConfig.DefaultOperator,
+ FuzzyMinSim = LuceneConfig.MinimumSimilarity
+ };
+
+ Query query = parser.Parse(LuceneSearchUtilityNodeAutocomplete.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm));
+ TopDocs topDocs = LuceneSearchUtilityNodeAutocomplete.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount);
+
+ for (int i = 0; i < topDocs.ScoreDocs.Length; i++)
+ {
+ // read back a Lucene doc from results
+ Document resultDoc = LuceneSearchUtilityNodeAutocomplete.Searcher.Doc(topDocs.ScoreDocs[i].Doc);
+
+ string name = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Name));
+ string docName = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.DocName));
+ string cat = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName));
+ string parameters = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Parameters));
+
+
+ var foundNode = FindViewModelForNodeNameAndCategory(name, cat, parameters);
+ if (foundNode != null)
+ {
+ candidates.Add(foundNode);
+ }
+ }
+
+ return candidates;
+ }
+ else
+ {
+ return Search(search);
+ }
+ }
+
///
/// Filters the matching node search elements based on user input in the search field.
///
@@ -617,9 +678,25 @@ internal void SearchAutoCompleteCandidates(string input)
}
else
{
- // Providing the saved search results to limit the scope of the query search.
- // Then add back the ML info on filterted nodes as the Search function accepts elements of type NodeSearchElement
- var foundNodes = Search(input, searchElementsCache.Select(x => x.Model));
+ LuceneSearchUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model);
+
+ //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created
+ LuceneSearchUtilityNodeAutocomplete.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
+
+ //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search)
+ foreach (var node in searchElementsCache.Select(x => x.Model))
+ {
+ var doc = LuceneSearchUtilityNodeAutocomplete.InitializeIndexDocumentForNodes();
+ AddNodeTypeToSearchIndex(node, doc);
+ }
+
+ //Write the Lucene documents to memory
+ LuceneSearchUtilityNodeAutocomplete.CommitWriterChanges();
+
+ var luceneResults = SearchNodeAutocomplete(input, true);
+ var foundNodesModels = luceneResults.Select(x => x.Model);
+ var foundNodes = foundNodesModels.Select(MakeNodeSearchElementVM);
+
var filteredSearchElements = new List();
foreach (var node in foundNodes)
@@ -635,10 +712,30 @@ internal void SearchAutoCompleteCandidates(string input)
}
}
FilteredResults = new List(filteredSearchElements).OrderBy(x => x.Name).ThenBy(x => x.Description);
+
+ LuceneSearchUtilityNodeAutocomplete.DisposeWriter();
}
}
}
+ ///
+ /// Add node information to Lucene index
+ ///
+ /// node info that will be indexed
+ /// Lucene document in which the node info will be indexed
+ private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
+ {
+ if (LuceneSearchUtilityNodeAutocomplete.addedFields == null) return;
+
+ LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
+ LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
+ LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
+ if (node.SearchKeywords.Count > 0) LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
+ LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);
+
+ LuceneSearchUtilityNodeAutocomplete.writer?.AddDocument(doc);
+ }
+
///
/// Returns a collection of node search elements for nodes
/// that output a type compatible with the port type if it's an input port.
diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs
index 9f5aaaea7ce..c3a1a0f7a2f 100644
--- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs
+++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs
@@ -995,7 +995,7 @@ internal IEnumerable Search(string search, bool useL
/// Full Category of the node
/// Node input parameters
///
- private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
+ internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
{
var result = Model.SearchEntries.Where(e => {
if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory))
@@ -1034,7 +1034,7 @@ private static IEnumerable GetVisibleSearchResults(N
}
}
- private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
+ internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
{
var element = entry as CustomNodeSearchElement;
var elementVM = element != null