diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index d40097b681b..982e1d0c508 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -21,6 +21,16 @@ internal class LuceneSearchUtility internal Lucene.Net.Store.Directory indexDir; internal IndexWriter writer; internal string directory; + internal LuceneStorage currentStorageType; + + public enum LuceneStorage + { + //Lucene Storage will be located in RAM and all the info indexed will be lost when Dynamo app is closed + RAM, + + //Lucene Storage will be located in the local File System and the files will remain in ...AppData\Roaming\Dynamo\Dynamo Core\2.19\Index folder + FILE_SYSTEM + } // Used for creating the StandardAnalyzer internal Analyzer Analyzer; @@ -36,23 +46,34 @@ internal LuceneSearchUtility(DynamoModel model) /// /// Initialize Lucene config file writer. /// - internal void InitializeLuceneConfig(string dirName) + internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM) { addedFields = new List(); - DirectoryInfo webBrowserUserDataFolder; + DirectoryInfo luceneUserDataFolder; var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); - webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; + luceneUserDataFolder = userDataDir.Exists ? userDataDir : null; directory = dirName; - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName); - indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); + string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName); + + currentStorageType = storageType; + + if (storageType == LuceneStorage.RAM) + { + indexDir = new RAMDirectory(); + } + else + { + indexDir = FSDirectory.Open(indexPath); + } + // Create an analyzer to process the text Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); - // Initialize Lucene index writer, unless in test mode. - if (!DynamoModel.IsTestMode) + // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info. + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { // Create an index writer IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) @@ -77,7 +98,7 @@ internal void InitializeLuceneConfig(string dirName) /// internal Document InitializeIndexDocumentForNodes() { - if (DynamoModel.IsTestMode) return null; + if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null; var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES); var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES); @@ -153,7 +174,7 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo ((StringField)doc.GetField(field)).SetStringValue(value); } - if (isLast && indexedFields.Any()) + if (isLast && indexedFields != null && indexedFields.Any()) { List diff = indexedFields.Except(addedFields).ToList(); foreach (var d in diff) @@ -248,7 +269,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm) internal void DisposeWriter() { //We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location - if (!DynamoModel.IsTestMode) + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { writer?.Dispose(); writer = null; @@ -257,7 +278,7 @@ internal void DisposeWriter() internal void CommitWriterChanges() { - if (!DynamoModel.IsTestMode) + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { //Commit the info indexed writer?.Commit(); diff --git a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs index 66748db5eb3..68677882d28 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs @@ -18,6 +18,9 @@ using Dynamo.Utilities; using Dynamo.Wpf.ViewModels; using Greg; +using Lucene.Net.Documents; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; using Newtonsoft.Json; using ProtoCore.AST.AssociativeAST; using ProtoCore.Mirror; @@ -39,6 +42,9 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel private bool displayLowConfidence; private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete"; + // Lucene search utility to perform indexing operations just for NodeAutocomplete. + internal LuceneSearchUtility LuceneSearchUtilityNodeAutocomplete { get; set; } + /// /// The Node AutoComplete ML service version, this could be empty if user has not used ML way /// @@ -602,6 +608,61 @@ private NodeSearchElementViewModel GetViewModelForNodeSearchElement(NodeSearchEl return null; } + + /// + /// Performs a search using the given string as query and subset, if provided. + /// + /// Returns a list with a maximum MaxNumSearchResults elements. + /// The search query + /// Temporary flag that will be used for searching using Lucene.NET + internal IEnumerable SearchNodeAutocomplete(string search, bool useLucene) + { + if (useLucene) + { + //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected + LuceneSearchUtilityNodeAutocomplete.dirReader = LuceneSearchUtilityNodeAutocomplete.writer?.GetReader(applyAllDeletes: true); + if (LuceneSearchUtilityNodeAutocomplete.dirReader == null) return null; + + LuceneSearchUtilityNodeAutocomplete.Searcher = new IndexSearcher(LuceneSearchUtilityNodeAutocomplete.dirReader); + + string searchTerm = search.Trim(); + var candidates = new List(); + var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtilityNodeAutocomplete.Analyzer) + { + AllowLeadingWildcard = true, + DefaultOperator = LuceneConfig.DefaultOperator, + FuzzyMinSim = LuceneConfig.MinimumSimilarity + }; + + Query query = parser.Parse(LuceneSearchUtilityNodeAutocomplete.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm)); + TopDocs topDocs = LuceneSearchUtilityNodeAutocomplete.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); + + for (int i = 0; i < topDocs.ScoreDocs.Length; i++) + { + // read back a Lucene doc from results + Document resultDoc = LuceneSearchUtilityNodeAutocomplete.Searcher.Doc(topDocs.ScoreDocs[i].Doc); + + string name = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Name)); + string docName = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.DocName)); + string cat = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName)); + string parameters = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Parameters)); + + + var foundNode = FindViewModelForNodeNameAndCategory(name, cat, parameters); + if (foundNode != null) + { + candidates.Add(foundNode); + } + } + + return candidates; + } + else + { + return Search(search); + } + } + /// /// Filters the matching node search elements based on user input in the search field. /// @@ -617,9 +678,25 @@ internal void SearchAutoCompleteCandidates(string input) } else { - // Providing the saved search results to limit the scope of the query search. - // Then add back the ML info on filterted nodes as the Search function accepts elements of type NodeSearchElement - var foundNodes = Search(input, searchElementsCache.Select(x => x.Model)); + LuceneSearchUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model); + + //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created + LuceneSearchUtilityNodeAutocomplete.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM); + + //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search) + foreach (var node in searchElementsCache.Select(x => x.Model)) + { + var doc = LuceneSearchUtilityNodeAutocomplete.InitializeIndexDocumentForNodes(); + AddNodeTypeToSearchIndex(node, doc); + } + + //Write the Lucene documents to memory + LuceneSearchUtilityNodeAutocomplete.CommitWriterChanges(); + + var luceneResults = SearchNodeAutocomplete(input, true); + var foundNodesModels = luceneResults.Select(x => x.Model); + var foundNodes = foundNodesModels.Select(MakeNodeSearchElementVM); + var filteredSearchElements = new List(); foreach (var node in foundNodes) @@ -635,10 +712,30 @@ internal void SearchAutoCompleteCandidates(string input) } } FilteredResults = new List(filteredSearchElements).OrderBy(x => x.Name).ThenBy(x => x.Description); + + LuceneSearchUtilityNodeAutocomplete.DisposeWriter(); } } } + /// + /// Add node information to Lucene index + /// + /// node info that will be indexed + /// Lucene document in which the node info will be indexed + private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) + { + if (LuceneSearchUtilityNodeAutocomplete.addedFields == null) return; + + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description); + if (node.SearchKeywords.Count > 0) LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty); + + LuceneSearchUtilityNodeAutocomplete.writer?.AddDocument(doc); + } + /// /// Returns a collection of node search elements for nodes /// that output a type compatible with the port type if it's an input port. diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index 9f5aaaea7ce..c3a1a0f7a2f 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -995,7 +995,7 @@ internal IEnumerable Search(string search, bool useL /// Full Category of the node /// Node input parameters /// - private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters) + internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters) { var result = Model.SearchEntries.Where(e => { if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory)) @@ -1034,7 +1034,7 @@ private static IEnumerable GetVisibleSearchResults(N } } - private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry) + internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry) { var element = entry as CustomNodeSearchElement; var elementVM = element != null