From 736b8a8ed98a8e411b8bf239da39a8872cd81a5d Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Wed, 21 Jun 2023 07:23:01 -0400 Subject: [PATCH 01/10] Lucene search in package manager --- src/DynamoCore/Configuration/LuceneConfig.cs | 16 +- src/DynamoCore/Models/DynamoModel.cs | 2 +- .../PackageManagerSearchViewModel.cs | 311 +++++++++++++++++- 3 files changed, 325 insertions(+), 4 deletions(-) diff --git a/src/DynamoCore/Configuration/LuceneConfig.cs b/src/DynamoCore/Configuration/LuceneConfig.cs index 226a6062052..32a92351698 100644 --- a/src/DynamoCore/Configuration/LuceneConfig.cs +++ b/src/DynamoCore/Configuration/LuceneConfig.cs @@ -107,7 +107,12 @@ public enum IndexFieldsEnum /// /// Documentation - Documentation of the node /// - Documentation + Documentation, + + /// + /// Hosts - Package hosts + /// + Hosts } /// @@ -119,5 +124,14 @@ public enum IndexFieldsEnum nameof(IndexFieldsEnum.SearchKeywords), nameof(IndexFieldsEnum.DocName), nameof(IndexFieldsEnum.Documentation)}; + + + /// + /// Package Fields to be indexed by Lucene Search + /// + public static string[] PackageIndexFields = { nameof(IndexFieldsEnum.Name), + nameof(IndexFieldsEnum.Description), + nameof(IndexFieldsEnum.SearchKeywords), + nameof(IndexFieldsEnum.Hosts)}; } } diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index 7d2816dd457..7c3785eb46f 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -628,7 +628,7 @@ private void InitializeLuceneConfig() var userDataDir = new DirectoryInfo(pathManager.UserDataDirectory); webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index"); + string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", "Nodes"); indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); // Create an analyzer to process the text diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index f29c36d91b7..a16a344fa42 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -3,18 +3,27 @@ using System.Collections.ObjectModel; using System.Collections.Specialized; using System.ComponentModel; +using System.IO; using System.Linq; using System.Threading.Tasks; using System.Windows; using System.Windows.Input; +using Dynamo.Configuration; using Dynamo.Interfaces; using Dynamo.Logging; +using Dynamo.Models; using Dynamo.PackageManager.ViewModels; using Dynamo.Search; using Dynamo.ViewModels; using Dynamo.Wpf.Properties; using Dynamo.Wpf.Utilities; using Greg.Responses; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; using Microsoft.Practices.Prism.Commands; using Microsoft.Practices.Prism.ViewModel; @@ -42,7 +51,7 @@ public enum PackageSortingKey Name, Downloads, Votes, - Maintainers, + Maintainers, LastUpdate, Search }; @@ -135,6 +144,16 @@ private void SetFilterHosts(object obj) } #region Properties & Fields + internal List addedFields; + internal DirectoryReader dirReader; + internal Lucene.Net.Store.Directory indexDir; + internal IndexWriter writer; + + // Holds the instance for the IndexSearcher + internal IndexSearcher Searcher; + + // Used for creating the StandardAnalyzer + internal Analyzer Analyzer; // The results of the last synchronization with the package manager server public List LastSync { get; set; } @@ -437,6 +456,113 @@ internal PackageManagerSearchViewModel() SelectedHosts = new List(); } + /// + /// Initialize Lucene config file writer. + /// + private void InitializeLuceneConfig() + { + addedFields = new List(); + + var dynamoModel = PackageManagerClientViewModel.DynamoViewModel.Model; + + DirectoryInfo webBrowserUserDataFolder; + var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); + webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; + + string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", "Packages"); + indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); + + // Create an analyzer to process the text + Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); + + // Initialize Lucene index writer, unless in test mode. + if (!DynamoModel.IsTestMode) + { + // Create an index writer + IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) + { + OpenMode = OpenMode.CREATE + }; + writer = new IndexWriter(indexDir, indexConfig); + } + } + + /// + /// Initialize Lucene index document object for reuse + /// + /// + private Document InitializeIndexDocument() + { + if (DynamoModel.IsTestMode) return null; + + var name = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Name), string.Empty, Field.Store.YES); + var description = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Description), string.Empty, Field.Store.YES); + var keywords = new TextField(nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), string.Empty, Field.Store.YES); + var hosts = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Hosts), string.Empty, Field.Store.YES); + + var d = new Document() + { + name, description, keywords, hosts + }; + return d; + } + + /// + /// Add package information to Lucene index + /// + /// package info that will be indexed + /// Lucene document in which the package info will be indexed + private void AddPackageToSearchIndex(PackageManagerSearchElement package, Document doc) + { + if (DynamoModel.IsTestMode) return; + if (addedFields == null) return; + + SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Name), package.Name); + SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Description), package.Description); + + if (package.Keywords.Count() > 0) + { + SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), package.Keywords); + } + + if (package.Hosts != null && string.IsNullOrEmpty(package.Hosts.ToString())) + { + SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Hosts), package.Hosts.ToString(), true, true); + } + + writer?.AddDocument(doc); + } + + /// + /// The SetDocumentFieldValue method should be optimized later + /// + /// Lucene document in which the package info is stored + /// Package field that is being updated in the document + /// Package field value + /// This is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. + /// This is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. + private void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) + { + addedFields.Add(field); + if (isTextField && !field.Equals("DocName")) + { + ((TextField)doc.GetField(field)).SetStringValue(value); + } + else + { + ((StringField)doc.GetField(field)).SetStringValue(value); + } + if (isLast) + { + List diff = LuceneConfig.PackageIndexFields.Except(addedFields).ToList(); + foreach (var d in diff) + { + SetDocumentFieldValue(doc, d, ""); + } + addedFields.Clear(); + } + } + /// /// The class constructor. /// @@ -444,6 +570,7 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi { PackageManagerClientViewModel = client; HostFilter = InitializeHostFilter(); + InitializeLuceneConfig(); } /// @@ -722,6 +849,8 @@ public void RefreshAndSearchAsync() this.ClearSearchResults(); this.SearchState = PackageSearchState.Syncing; + var iDoc = InitializeIndexDocument(); + Task>.Factory.StartNew(RefreshAndSearch).ContinueWith((t) => { lock (SearchResults) @@ -729,9 +858,24 @@ public void RefreshAndSearchAsync() ClearSearchResults(); foreach (var result in t.Result) { + if (result.Model != null) + { + AddPackageToSearchIndex(result.Model, iDoc); + } this.AddToSearchResults(result); } this.SearchState = HasNoResults ? PackageSearchState.NoResults : PackageSearchState.Results; + + if (!DynamoModel.IsTestMode) + { + dirReader = writer?.GetReader(applyAllDeletes: true); + Searcher = new IndexSearcher(dirReader); + + writer?.Commit(); + writer?.Dispose(); + indexDir?.Dispose(); + writer = null; + } } RefreshInfectedPackages(); } @@ -879,9 +1023,19 @@ internal void SearchAndUpdateResults(string query) // if last sync isn't populated, we can't search if (LastSync == null) return; + IEnumerable results; this.SearchText = query; - var results = Search(query); + // If the search query is empty, just call regular search API + // If the search query is not empty, then call Lucene search API + if (string.IsNullOrEmpty(query)) + { + results = Search(query); + } + else + { + results = Search(query, true); + } this.ClearSearchResults(); @@ -1005,6 +1159,159 @@ internal IEnumerable Search(string query) return list; } + /// + /// Performs a search using the given string as query, but does not update + /// the SearchResults object. + /// + /// Returns a list with a maximum MaxNumSearchResults elements. + /// The search query + /// /// Temporary flag that will be used for searching using Lucene.NET + internal IEnumerable Search(string searchText, bool useLucene) + { + if (useLucene) + { + string searchTerm = searchText.Trim(); + var packages = new List(); + + string[] fnames = LuceneConfig.PackageIndexFields; + + var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, fnames, Analyzer) + { + AllowLeadingWildcard = true, + DefaultOperator = LuceneConfig.DefaultOperator, + FuzzyMinSim = LuceneConfig.MinimumSimilarity + }; + + Query query = parser.Parse(CreateSearchQuery(fnames, searchTerm)); + + //indicate we want the first 50 results + TopDocs topDocs = Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); + for (int i = 0; i < topDocs.ScoreDocs.Length; i++) + { + //read back a doc from results + Document resultDoc = Searcher.Doc(topDocs.ScoreDocs[i].Doc); + + // Get the view model of the package element and add it to the results. + string name = resultDoc.Get(nameof(LuceneConfig.IndexFieldsEnum.Name)); + + var foundPackage = GetViewModelForPackageSearchElement(name); + if (foundPackage != null) + { + packages.Add(foundPackage); + } + } + return packages; + } + else + { + return Search(searchText); + } + } + + /// + /// To get view model for a package based on its name. + /// s + /// Name of the package + /// + private PackageManagerSearchElementViewModel GetViewModelForPackageSearchElement(string packageName) + { + var result = PackageManagerClientViewModel.CachedPackageList.Where(e => { + if (e.Name.Equals(packageName)) + { + return true; + } + return false; + }); + + if (!result.Any()) + { + return null; + } + + return new PackageManagerSearchElementViewModel(result.ElementAt(0), false); + } + + /// + /// Creates a search query with adjusted priority, fuzzy logic and wildcards. + /// Complete Search term appearing in Name of the package will be given highest priority. + /// Then, complete search term appearing in other metadata, + /// Then, a part of the search term(if containing multiple words) appearing in Name of the package + /// Then, a part of the search term appearing in other metadata of the package. + /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. + /// Then, the same fuzzy logic will be applied to each part of the search term. + /// + /// All fields to be searched in. + /// Search key to be searched for. + /// + private string CreateSearchQuery(string[] fields, string SearchTerm) + { + int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; + // Use a larger max edit value - more tolerant with typo when search term is longer than threshold + if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) + { + fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; + } + + var booleanQuery = new BooleanQuery(); + string searchTerm = QueryParser.Escape(SearchTerm); + + foreach (string f in fields) + { + FuzzyQuery fuzzyQuery; + if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + + var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.SearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + if (searchTerm.Contains(' ') || searchTerm.Contains('.')) + { + foreach (string s in searchTerm.Split(' ', '.')) + { + if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); + + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = 5; + } + else + { + wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + } + } + } + return booleanQuery.ToString(); + } + /// /// Sort a list of search results by the given key /// From 2ed92f6e48bd46c476a8d2bf25df05e838be1ec2 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Fri, 23 Jun 2023 10:12:19 -0400 Subject: [PATCH 02/10] Move to LuceneSearchViewModel --- src/DynamoCore/Configuration/LuceneConfig.cs | 10 ++ src/DynamoCore/Models/DynamoModel.cs | 2 +- src/DynamoCoreWpf/DynamoCoreWpf.csproj | 1 + .../Lucene/LuceneSearchViewModel.cs | 100 ++++++++++++++++++ .../PackageManagerSearchViewModel.cs | 89 ++-------------- .../ViewModels/Search/SearchViewModel.cs | 88 +-------------- 6 files changed, 123 insertions(+), 167 deletions(-) create mode 100644 src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs diff --git a/src/DynamoCore/Configuration/LuceneConfig.cs b/src/DynamoCore/Configuration/LuceneConfig.cs index 32a92351698..cf8279df164 100644 --- a/src/DynamoCore/Configuration/LuceneConfig.cs +++ b/src/DynamoCore/Configuration/LuceneConfig.cs @@ -74,6 +74,16 @@ internal class LuceneConfig /// internal static int FuzzySearchWeight = 2; + /// + /// Directory where Nodes info are indexed + /// + internal static string NodesIndexingDirectory = "Nodes"; + + /// + /// Directory where packages info are indexed + /// + internal static string PackagesIndexingDirectory = "Packages"; + /// /// This represent the fields that will be indexed when initializing Lucene Search /// diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index 7c3785eb46f..e6ee74aff85 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -628,7 +628,7 @@ private void InitializeLuceneConfig() var userDataDir = new DirectoryInfo(pathManager.UserDataDirectory); webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", "Nodes"); + string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", LuceneConfig.NodesIndexingDirectory); indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); // Create an analyzer to process the text diff --git a/src/DynamoCoreWpf/DynamoCoreWpf.csproj b/src/DynamoCoreWpf/DynamoCoreWpf.csproj index 0a8e00fdd71..40d9ac1a54b 100644 --- a/src/DynamoCoreWpf/DynamoCoreWpf.csproj +++ b/src/DynamoCoreWpf/DynamoCoreWpf.csproj @@ -521,6 +521,7 @@ + diff --git a/src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs new file mode 100644 index 00000000000..5de807f28dc --- /dev/null +++ b/src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs @@ -0,0 +1,100 @@ +using System.Linq; +using Dynamo.Configuration; +using Dynamo.Models; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; + +namespace Dynamo.ViewModels +{ + internal class LuceneSearchViewModel + { + internal DynamoModel dynamoModel; + + internal LuceneSearchViewModel(DynamoModel model) + { + dynamoModel = model; + } + + /// + /// Creates a search query with adjusted priority, fuzzy logic and wildcards. + /// Complete Search term appearing in Name of the package will be given highest priority. + /// Then, complete search term appearing in other metadata, + /// Then, a part of the search term(if containing multiple words) appearing in Name of the package + /// Then, a part of the search term appearing in other metadata of the package. + /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. + /// Then, the same fuzzy logic will be applied to each part of the search term. + /// + /// All fields to be searched in. + /// Search key to be searched for. + /// + internal string CreateSearchQuery(string[] fields, string SearchTerm) + { + int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; + // Use a larger max edit value - more tolerant with typo when search term is longer than threshold + if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) + { + fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; + } + + var booleanQuery = new BooleanQuery(); + string searchTerm = QueryParser.Escape(SearchTerm); + + foreach (string f in fields) + { + FuzzyQuery fuzzyQuery; + if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + + var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.SearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + if (searchTerm.Contains(' ') || searchTerm.Contains('.')) + { + foreach (string s in searchTerm.Split(' ', '.')) + { + if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); + + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = 5; + } + else + { + wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + } + } + } + return booleanQuery.ToString(); + } + } +} diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index a16a344fa42..69e73bb6e41 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -155,6 +155,9 @@ private void SetFilterHosts(object obj) // Used for creating the StandardAnalyzer internal Analyzer Analyzer; + // Lucene search view model. + internal LuceneSearchViewModel LuceneSearchViewModel { get; set; } + // The results of the last synchronization with the package manager server public List LastSync { get; set; } @@ -469,7 +472,7 @@ private void InitializeLuceneConfig() var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", "Packages"); + string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", LuceneConfig.PackagesIndexingDirectory); indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); // Create an analyzer to process the text @@ -570,6 +573,7 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi { PackageManagerClientViewModel = client; HostFilter = InitializeHostFilter(); + LuceneSearchViewModel = new LuceneSearchViewModel(PackageManagerClientViewModel.DynamoViewModel.Model); InitializeLuceneConfig(); } @@ -1182,7 +1186,7 @@ internal IEnumerable Search(string searchT FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(CreateSearchQuery(fnames, searchTerm)); + Query query = parser.Parse(LuceneSearchViewModel.CreateSearchQuery(fnames, searchTerm)); //indicate we want the first 50 results TopDocs topDocs = Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); @@ -1231,87 +1235,6 @@ private PackageManagerSearchElementViewModel GetViewModelForPackageSearchElement return new PackageManagerSearchElementViewModel(result.ElementAt(0), false); } - /// - /// Creates a search query with adjusted priority, fuzzy logic and wildcards. - /// Complete Search term appearing in Name of the package will be given highest priority. - /// Then, complete search term appearing in other metadata, - /// Then, a part of the search term(if containing multiple words) appearing in Name of the package - /// Then, a part of the search term appearing in other metadata of the package. - /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. - /// Then, the same fuzzy logic will be applied to each part of the search term. - /// - /// All fields to be searched in. - /// Search key to be searched for. - /// - private string CreateSearchQuery(string[] fields, string SearchTerm) - { - int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; - // Use a larger max edit value - more tolerant with typo when search term is longer than threshold - if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) - { - fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; - } - - var booleanQuery = new BooleanQuery(); - string searchTerm = QueryParser.Escape(SearchTerm); - - foreach (string f in fields) - { - FuzzyQuery fuzzyQuery; - if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - - var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.SearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - if (searchTerm.Contains(' ') || searchTerm.Contains('.')) - { - foreach (string s in searchTerm.Split(' ', '.')) - { - if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); - - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = 5; - } - else - { - wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - } - } - } - return booleanQuery.ToString(); - } - /// /// Sort a list of search results by the given key /// diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index 75d185c987c..e8f8abb917f 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -20,7 +20,6 @@ using Dynamo.Wpf.Services; using Dynamo.Wpf.ViewModels; using Lucene.Net.Documents; -using Lucene.Net.Index; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; @@ -350,6 +349,9 @@ public NodeSearchElementViewModel FindViewModelForNode(string nodeName) public NodeSearchModel Model { get; private set; } internal readonly DynamoViewModel dynamoViewModel; + // Lucene search view model. + internal LuceneSearchViewModel LuceneSearchViewModel { get; set; } + /// /// Class name, that has been clicked in library search view. /// @@ -370,6 +372,7 @@ internal SearchViewModel(DynamoViewModel dynamoViewModel) iconServices = new IconServices(pathManager); InitializeCore(); + LuceneSearchViewModel = new LuceneSearchViewModel(dynamoViewModel.Model); } // Just for tests. Please refer to LibraryTests.cs @@ -945,7 +948,7 @@ internal IEnumerable Search(string search, bool useL FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(CreateSearchQuery(LuceneConfig.IndexFields, searchTerm)); + Query query = parser.Parse(LuceneSearchViewModel.CreateSearchQuery(LuceneConfig.IndexFields, searchTerm)); TopDocs topDocs = Model.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); for (int i = 0; i < topDocs.ScoreDocs.Length; i++) @@ -978,87 +981,6 @@ internal IEnumerable Search(string search, bool useL } } - /// - /// Creates a search query with adjusted priority, fuzzy logic and wildcards. - /// Complete Search term appearing in Name of the node will be given highest priority. - /// Then, complete search term appearing in other metadata, - /// Then, a part of the search term(if containing multiple words) appearing in Name of the node - /// Then, a part of the search term appearing in other metadata of the node. - /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. - /// Then, the same fuzzy logic will be applied to each part of the search term. - /// - /// All fields to be searched in. - /// Search key to be searched for. - /// - private string CreateSearchQuery(string[] fields, string SearchTerm) - { - int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; - // Use a larger max edit value - more tolerant with typo when search term is longer than threshold - if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) - { - fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; - } - - var booleanQuery = new BooleanQuery(); - string searchTerm = QueryParser.Escape(SearchTerm); - - foreach (string f in fields) - { - FuzzyQuery fuzzyQuery; - if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - - var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.SearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - if (searchTerm.Contains(' ') || searchTerm.Contains('.')) - { - foreach (string s in searchTerm.Split(' ', '.')) - { - if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); - - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = 5; - } - else - { - wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - } - } - } - return booleanQuery.ToString(); - } - /// /// To get view model for a node based on its name and category /// From d6fe553d2c23eeb0af8d41d65dd966643fcce867 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:16:22 -0400 Subject: [PATCH 03/10] LuceneSearchUtility --- src/DynamoCoreWpf/DynamoCoreWpf.csproj | 2 +- .../LuceneSearchUtility.cs} | 6 +++--- .../PackageManager/PackageManagerSearchViewModel.cs | 7 ++++--- src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) rename src/DynamoCoreWpf/{ViewModels/Lucene/LuceneSearchViewModel.cs => Utilities/LuceneSearchUtility.cs} (97%) diff --git a/src/DynamoCoreWpf/DynamoCoreWpf.csproj b/src/DynamoCoreWpf/DynamoCoreWpf.csproj index 40d9ac1a54b..9fd98c66fc7 100644 --- a/src/DynamoCoreWpf/DynamoCoreWpf.csproj +++ b/src/DynamoCoreWpf/DynamoCoreWpf.csproj @@ -269,6 +269,7 @@ + @@ -521,7 +522,6 @@ - diff --git a/src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs b/src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs similarity index 97% rename from src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs rename to src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs index 5de807f28dc..fba62e40aea 100644 --- a/src/DynamoCoreWpf/ViewModels/Lucene/LuceneSearchViewModel.cs +++ b/src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs @@ -5,13 +5,13 @@ using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; -namespace Dynamo.ViewModels +namespace Dynamo.Utilities { - internal class LuceneSearchViewModel + internal class LuceneSearchUtility { internal DynamoModel dynamoModel; - internal LuceneSearchViewModel(DynamoModel model) + internal LuceneSearchUtility(DynamoModel model) { dynamoModel = model; } diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index 69e73bb6e41..0defd7a6c00 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -14,6 +14,7 @@ using Dynamo.Models; using Dynamo.PackageManager.ViewModels; using Dynamo.Search; +using Dynamo.Utilities; using Dynamo.ViewModels; using Dynamo.Wpf.Properties; using Dynamo.Wpf.Utilities; @@ -156,7 +157,7 @@ private void SetFilterHosts(object obj) internal Analyzer Analyzer; // Lucene search view model. - internal LuceneSearchViewModel LuceneSearchViewModel { get; set; } + internal LuceneSearchUtility LuceneSearchUtility { get; set; } // The results of the last synchronization with the package manager server public List LastSync { get; set; } @@ -573,7 +574,7 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi { PackageManagerClientViewModel = client; HostFilter = InitializeHostFilter(); - LuceneSearchViewModel = new LuceneSearchViewModel(PackageManagerClientViewModel.DynamoViewModel.Model); + LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model); InitializeLuceneConfig(); } @@ -1186,7 +1187,7 @@ internal IEnumerable Search(string searchT FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(LuceneSearchViewModel.CreateSearchQuery(fnames, searchTerm)); + Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(fnames, searchTerm)); //indicate we want the first 50 results TopDocs topDocs = Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index e8f8abb917f..aeeab593e90 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -350,7 +350,7 @@ public NodeSearchElementViewModel FindViewModelForNode(string nodeName) internal readonly DynamoViewModel dynamoViewModel; // Lucene search view model. - internal LuceneSearchViewModel LuceneSearchViewModel { get; set; } + internal LuceneSearchUtility LuceneSearchUtility { get; set; } /// /// Class name, that has been clicked in library search view. @@ -372,7 +372,7 @@ internal SearchViewModel(DynamoViewModel dynamoViewModel) iconServices = new IconServices(pathManager); InitializeCore(); - LuceneSearchViewModel = new LuceneSearchViewModel(dynamoViewModel.Model); + LuceneSearchUtility = new LuceneSearchUtility(dynamoViewModel.Model); } // Just for tests. Please refer to LibraryTests.cs @@ -948,7 +948,7 @@ internal IEnumerable Search(string search, bool useL FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(LuceneSearchViewModel.CreateSearchQuery(LuceneConfig.IndexFields, searchTerm)); + Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(LuceneConfig.IndexFields, searchTerm)); TopDocs topDocs = Model.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); for (int i = 0; i < topDocs.ScoreDocs.Length; i++) From 25f9cd076f194c483ebbef89d3fa8732c0424cb2 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Mon, 26 Jun 2023 18:46:05 -0400 Subject: [PATCH 04/10] Obsolete the search method in package manager view model. --- .../PackageManagerSearchViewModel.cs | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index 0defd7a6c00..f067250c629 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -846,7 +846,7 @@ public void Refresh() public IEnumerable RefreshAndSearch() { Refresh(); - return Search(SearchText); + return GetAllPackages(); } public void RefreshAndSearchAsync() @@ -1035,7 +1035,7 @@ internal void SearchAndUpdateResults(string query) // If the search query is not empty, then call Lucene search API if (string.IsNullOrEmpty(query)) { - results = Search(query); + results = GetAllPackages(); } else { @@ -1119,12 +1119,45 @@ internal IEnumerable Filter(IEnumerable + /// Get all the package results in the package manager. + /// + /// Returns a list with a maximum MaxNumSearchResults elements. + internal IEnumerable GetAllPackages() + { + if (LastSync == null) return new List(); + + List list = null; + + var isEnabledForInstall = !(Preferences as IDisablePackageLoadingPreferences).DisableCustomPackageLocations; + + // Don't show deprecated packages + list = Filter(LastSync.Where(x => !x.IsDeprecated) + .Select(x => new PackageManagerSearchElementViewModel(x, + PackageManagerClientViewModel.AuthenticationManager.HasAuthProvider, + CanInstallPackage(x.Name), isEnabledForInstall))) + .ToList(); + + Sort(list, this.SortingKey); + + if (SortingDirection == PackageSortingDirection.Descending) + { + list.Reverse(); + } + + foreach (var x in list) + x.RequestShowFileDialog += OnRequestShowFileDialog; + + return list; + } + /// /// Performs a search using the given string as query, but does not update /// the SearchResults object. /// /// Returns a list with a maximum MaxNumSearchResults elements. /// The search query + [Obsolete("This method will be removed in future Dynamo versions - please use Search method with Lucene flag.")] internal IEnumerable Search(string query) { if (LastSync == null) return new List(); @@ -1209,7 +1242,7 @@ internal IEnumerable Search(string searchT } else { - return Search(searchText); + return GetAllPackages(); } } From 2707baee43d5955a89a2339bb075e24b5ee839e8 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Mon, 26 Jun 2023 18:56:22 -0400 Subject: [PATCH 05/10] Update PackageManagerClientViewModel.cs --- .../ViewModels/PackageManager/PackageManagerClientViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs index 00178196ffb..aa40194de8a 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs @@ -491,7 +491,7 @@ public List GetInfectedPackages() { InfectedPackageList = new List(); var latestPkgs = Model.GetUsersLatestPackages(); - if (latestPkgs != null && latestPkgs.maintains.Count > 0) + if (latestPkgs != null && latestPkgs.maintains?.Count > 0) { foreach (var infectedVer in latestPkgs.maintains) { From 0d879a2e1a640025ed8de37f4cdac60a24b36d5e Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:37:41 -0400 Subject: [PATCH 06/10] Move Lucene methods into a dynamo core utility --- src/DynamoCore/Configuration/LuceneConfig.cs | 19 +- src/DynamoCore/Models/DynamoModel.cs | 126 ++-------- src/DynamoCore/Search/NodeSearchModel.cs | 11 +- .../Utilities/LuceneSearchUtility.cs | 237 ++++++++++++++++++ src/DynamoCoreWpf/DynamoCoreWpf.csproj | 1 - .../Utilities/LuceneSearchUtility.cs | 100 -------- .../PackageManagerSearchViewModel.cs | 134 ++-------- .../ViewModels/Search/SearchViewModel.cs | 10 +- 8 files changed, 294 insertions(+), 344 deletions(-) create mode 100644 src/DynamoCore/Utilities/LuceneSearchUtility.cs delete mode 100644 src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs diff --git a/src/DynamoCore/Configuration/LuceneConfig.cs b/src/DynamoCore/Configuration/LuceneConfig.cs index cf8279df164..45aaec84a96 100644 --- a/src/DynamoCore/Configuration/LuceneConfig.cs +++ b/src/DynamoCore/Configuration/LuceneConfig.cs @@ -74,6 +74,11 @@ internal class LuceneConfig /// internal static int FuzzySearchWeight = 2; + /// + /// Parent directory where information is indexed. + /// + internal static string Index = "Index"; + /// /// Directory where Nodes info are indexed /// @@ -126,14 +131,14 @@ public enum IndexFieldsEnum } /// - /// Fields to be indexed by Lucene Search + /// Nodes Fields to be indexed by Lucene Search /// - public static string[] IndexFields = { nameof(IndexFieldsEnum.Name), - nameof(IndexFieldsEnum.FullCategoryName), - nameof(IndexFieldsEnum.Description), - nameof(IndexFieldsEnum.SearchKeywords), - nameof(IndexFieldsEnum.DocName), - nameof(IndexFieldsEnum.Documentation)}; + public static string[] NodeIndexFields = { nameof(IndexFieldsEnum.Name), + nameof(IndexFieldsEnum.FullCategoryName), + nameof(IndexFieldsEnum.Description), + nameof(IndexFieldsEnum.SearchKeywords), + nameof(IndexFieldsEnum.DocName), + nameof(IndexFieldsEnum.Documentation)}; /// diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index e6ee74aff85..0fbd586d46e 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -39,9 +39,7 @@ using Dynamo.Utilities; using DynamoServices; using Greg; -using Lucene.Net.Analysis.Standard; using Lucene.Net.Documents; -using Lucene.Net.Index; using Lucene.Net.Search; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -134,10 +132,6 @@ public partial class DynamoModel : IDynamoModel, IDisposable, IEngineControllerM private Timer backupFilesTimer; private Dictionary backupFilesDict = new Dictionary(); internal readonly Stopwatch stopwatch = Stopwatch.StartNew(); - internal IndexWriter writer; - internal Lucene.Net.Store.Directory indexDir; - internal DirectoryReader dirReader; - internal List addedFields; /// /// Indicating if ASM is loaded correctly, defaulting to true because integrators most likely have code for ASM preloading @@ -145,6 +139,9 @@ public partial class DynamoModel : IDynamoModel, IDisposable, IEngineControllerM /// internal bool IsASMLoaded = true; + // Lucene search utility to perform indexing operations. + internal LuceneSearchUtility LuceneSearchUtility { get; set; } + #endregion #region static properties @@ -619,34 +616,6 @@ public static DynamoModel Start(IStartConfiguration configuration) // Token representing the standard library directory internal static readonly string StandardLibraryToken = @"%StandardLibrary%"; - - private void InitializeLuceneConfig() - { - addedFields = new List(); - - DirectoryInfo webBrowserUserDataFolder; - var userDataDir = new DirectoryInfo(pathManager.UserDataDirectory); - webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; - - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", LuceneConfig.NodesIndexingDirectory); - indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); - - // Create an analyzer to process the text - SearchModel.Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); - - // When running parallel tests several are trying to write in the AppData folder then the job - // is failing and in a wrong state so we prevent to initialize Lucene index writer during test mode. - if (!IsTestMode) - { - // Create an index writer - IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, SearchModel.Analyzer) - { - OpenMode = OpenMode.CREATE - }; - writer = new IndexWriter(indexDir, indexConfig); - } - } - /// /// Default constructor for DynamoModel /// @@ -959,7 +928,8 @@ protected DynamoModel(IStartConfiguration config) CustomNodeManager = new CustomNodeManager(NodeFactory, MigrationManager, LibraryServices); - InitializeLuceneConfig(); + LuceneSearchUtility = new LuceneSearchUtility(this); + LuceneSearchUtility.InitializeLuceneConfig(LuceneConfig.NodesIndexingDirectory); InitializeCustomNodeManager(); @@ -1390,8 +1360,8 @@ public void Dispose() } // Lucene disposals (just if LuceneNET was initialized) - indexDir?.Dispose(); - dirReader?.Dispose(); + LuceneSearchUtility.indexDir?.Dispose(); + LuceneSearchUtility.dirReader?.Dispose(); #if DEBUG CurrentWorkspace.NodeAdded -= CrashOnDemand.CurrentWorkspace_NodeAdded; @@ -1475,7 +1445,7 @@ private void InitializeCustomNodeManager() private void InitializeIncludedNodes() { - var iDoc = InitializeIndexDocument(); + var iDoc = LuceneSearchUtility.InitializeIndexDocumentForNodes(); var customNodeData = new TypeLoadData(typeof(Function)); NodeFactory.AddLoader(new CustomNodeLoader(CustomNodeManager, IsTestMode)); @@ -1644,13 +1614,12 @@ private void InitializeNodeLibrary() // Without the index files on disk, the dirReader cant be initialized correctly. So does the searcher. if (!IsTestMode) { - dirReader = writer?.GetReader(applyAllDeletes: true); - IndexSearcher searcher = new IndexSearcher(dirReader); - SearchModel.Searcher = searcher; + LuceneSearchUtility.dirReader = LuceneSearchUtility.writer?.GetReader(applyAllDeletes: true); + LuceneSearchUtility.Searcher = new IndexSearcher(LuceneSearchUtility.dirReader); - writer?.Commit(); - writer?.Dispose(); - writer = null; + LuceneSearchUtility.writer?.Commit(); + LuceneSearchUtility.writer?.Dispose(); + LuceneSearchUtility.writer = null; } } @@ -1703,7 +1672,7 @@ internal void LoadNodeLibrary(Assembly assem, bool suppressZeroTouchLibraryLoad private void LoadNodeModels(List nodes, bool isPackageMember) { - var iDoc = InitializeIndexDocument(); + var iDoc = LuceneSearchUtility.InitializeIndexDocumentForNodes(); foreach (var type in nodes) { // Protect ourselves from exceptions thrown by malformed third party nodes. @@ -3266,33 +3235,6 @@ private NodeModelSearchElement AddNodeTypeToSearch(TypeLoadData typeLoadData) return node; } - /// - /// Initialize Lucene index document object for reuse - /// - /// - private Document InitializeIndexDocument() - { - if (IsTestMode) return null; - - var name = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Name), string.Empty, Field.Store.YES); - var fullCategory = new TextField(nameof(LuceneConfig.IndexFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES); - var description = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Description), string.Empty, Field.Store.YES); - var keywords = new TextField(nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), string.Empty, Field.Store.YES); - var docName = new StringField(nameof(LuceneConfig.IndexFieldsEnum.DocName), string.Empty, Field.Store.YES); - var fullDoc = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Documentation), string.Empty, Field.Store.YES); - - var d = new Document() - { - fullCategory, - name, - description, - keywords, - fullDoc, - docName - }; - return d; - } - /// /// Add node information to Lucene index /// @@ -3301,40 +3243,14 @@ private Document InitializeIndexDocument() private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) { if (IsTestMode) return; - if (addedFields == null) return; + if (LuceneSearchUtility.addedFields == null) return; - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.FullCategoryName), node.FullCategoryName); - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Name), node.Name); - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Description), node.Description); - if (node.SearchKeywords.Count > 0) SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.FullCategoryName), node.FullCategoryName); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Name), node.Name); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Description), node.Description); + if (node.SearchKeywords.Count > 0) LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); - writer?.AddDocument(doc); - } - - //TODO: - //isLast option is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. - //isTextField is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. - //The SetDocumentFieldValue method should be optimized later - private void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) - { - addedFields.Add(field); - if (isTextField && !field.Equals("DocName")) - { - ((TextField)doc.GetField(field)).SetStringValue(value); - } - else - { - ((StringField)doc.GetField(field)).SetStringValue(value); - } - if (isLast) - { - List diff = LuceneConfig.IndexFields.Except(addedFields).ToList(); - foreach (var d in diff) - { - SetDocumentFieldValue(doc, d, ""); - } - addedFields.Clear(); - } + LuceneSearchUtility.writer?.AddDocument(doc); } /// @@ -3365,7 +3281,7 @@ internal void HideUnhideNamespace(bool hide, string library, string namespc) internal void AddZeroTouchNodesToSearch(IEnumerable functionGroups) { - var iDoc = InitializeIndexDocument(); + var iDoc = LuceneSearchUtility.InitializeIndexDocumentForNodes(); foreach (var funcGroup in functionGroups) AddZeroTouchNodeToSearch(funcGroup, iDoc); } diff --git a/src/DynamoCore/Search/NodeSearchModel.cs b/src/DynamoCore/Search/NodeSearchModel.cs index 501c5b6ea11..6bdf761898a 100644 --- a/src/DynamoCore/Search/NodeSearchModel.cs +++ b/src/DynamoCore/Search/NodeSearchModel.cs @@ -4,11 +4,9 @@ using System.Xml; using Dynamo.Configuration; using Dynamo.Graph.Nodes; +using Dynamo.Logging; using Dynamo.Search.SearchElements; using DynamoUtilities; -using Dynamo.Logging; -using Lucene.Net.Search; -using Lucene.Net.Analysis; namespace Dynamo.Search { @@ -17,13 +15,6 @@ namespace Dynamo.Search /// public class NodeSearchModel : SearchLibrary { - - // Holds the instance for the IndexSearcher - internal IndexSearcher Searcher; - - // Used in DynamoModel for creating the StandardAnalyzer - internal Analyzer Analyzer; - /// /// Construct a NodeSearchModel object /// diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs new file mode 100644 index 00000000000..f09f2cff682 --- /dev/null +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -0,0 +1,237 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Dynamo.Configuration; +using Dynamo.Models; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; + +namespace Dynamo.Utilities +{ + internal class LuceneSearchUtility + { + internal DynamoModel dynamoModel; + internal List addedFields; + internal DirectoryReader dirReader; + internal Lucene.Net.Store.Directory indexDir; + internal IndexWriter writer; + internal string directory; + + // Used for creating the StandardAnalyzer + internal Analyzer Analyzer; + + // Holds the instance for the IndexSearcher + internal IndexSearcher Searcher; + + internal LuceneSearchUtility(DynamoModel model) + { + dynamoModel = model; + } + + /// + /// Initialize Lucene config file writer. + /// + internal void InitializeLuceneConfig(string dirName) + { + addedFields = new List(); + + DirectoryInfo webBrowserUserDataFolder; + var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); + webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; + + directory = dirName; + string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName); + indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); + + // Create an analyzer to process the text + Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); + + // Initialize Lucene index writer, unless in test mode. + if (!DynamoModel.IsTestMode) + { + // Create an index writer + IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) + { + OpenMode = OpenMode.CREATE + }; + writer = new IndexWriter(indexDir, indexConfig); + } + } + + /// + /// Initialize Lucene index document object for reuse + /// + /// + internal Document InitializeIndexDocumentForNodes() + { + if (DynamoModel.IsTestMode) return null; + + var name = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Name), string.Empty, Field.Store.YES); + var fullCategory = new TextField(nameof(LuceneConfig.IndexFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES); + var description = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Description), string.Empty, Field.Store.YES); + var keywords = new TextField(nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), string.Empty, Field.Store.YES); + var docName = new StringField(nameof(LuceneConfig.IndexFieldsEnum.DocName), string.Empty, Field.Store.YES); + var fullDoc = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Documentation), string.Empty, Field.Store.YES); + + var d = new Document() + { + fullCategory, + name, + description, + keywords, + fullDoc, + docName + }; + return d; + } + + /// + /// Initialize Lucene index document object for reuse + /// + /// + internal Document InitializeIndexDocumentForPackages() + { + if (DynamoModel.IsTestMode) return null; + + var name = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Name), string.Empty, Field.Store.YES); + var description = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Description), string.Empty, Field.Store.YES); + var keywords = new TextField(nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), string.Empty, Field.Store.YES); + var hosts = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Hosts), string.Empty, Field.Store.YES); + + var d = new Document() + { + name, description, keywords, hosts + }; + return d; + } + + // TODO: + // isLast option is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. + // isTextField is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. + /// + /// The SetDocumentFieldValue method should be optimized later + /// + /// Lucene document in which the information is stored + /// Field that is being updated in the document + /// Field value + /// This is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. + /// This is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. + internal void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) + { + string[] indexedFields = null; + if (directory.Equals(LuceneConfig.NodesIndexingDirectory)) + { + indexedFields = LuceneConfig.NodeIndexFields; + } + else if (directory.Equals(LuceneConfig.PackagesIndexingDirectory)) + { + indexedFields = LuceneConfig.PackageIndexFields; + } + + addedFields.Add(field); + if (isTextField && !field.Equals("DocName")) + { + ((TextField)doc.GetField(field)).SetStringValue(value); + } + else + { + ((StringField)doc.GetField(field)).SetStringValue(value); + } + + if (isLast && indexedFields.Any()) + { + List diff = indexedFields.Except(addedFields).ToList(); + foreach (var d in diff) + { + SetDocumentFieldValue(doc, d, ""); + } + addedFields.Clear(); + } + } + + /// + /// Creates a search query with adjusted priority, fuzzy logic and wildcards. + /// Complete Search term appearing in Name of the package will be given highest priority. + /// Then, complete search term appearing in other metadata, + /// Then, a part of the search term(if containing multiple words) appearing in Name of the package + /// Then, a part of the search term appearing in other metadata of the package. + /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. + /// Then, the same fuzzy logic will be applied to each part of the search term. + /// + /// All fields to be searched in. + /// Search key to be searched for. + /// + internal string CreateSearchQuery(string[] fields, string SearchTerm) + { + int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; + // Use a larger max edit value - more tolerant with typo when search term is longer than threshold + if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) + { + fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; + } + + var booleanQuery = new BooleanQuery(); + string searchTerm = QueryParser.Escape(SearchTerm); + + foreach (string f in fields) + { + FuzzyQuery fuzzyQuery; + if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + + var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.SearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; + } + else + { + wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + + if (searchTerm.Contains(' ') || searchTerm.Contains('.')) + { + foreach (string s in searchTerm.Split(' ', '.')) + { + if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) + { + fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); + booleanQuery.Add(fuzzyQuery, Occur.SHOULD); + } + wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); + + if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) + { + wildcardQuery.Boost = 5; + } + else + { + wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; + } + booleanQuery.Add(wildcardQuery, Occur.SHOULD); + } + } + } + return booleanQuery.ToString(); + } + } +} diff --git a/src/DynamoCoreWpf/DynamoCoreWpf.csproj b/src/DynamoCoreWpf/DynamoCoreWpf.csproj index 9fd98c66fc7..0a8e00fdd71 100644 --- a/src/DynamoCoreWpf/DynamoCoreWpf.csproj +++ b/src/DynamoCoreWpf/DynamoCoreWpf.csproj @@ -269,7 +269,6 @@ - diff --git a/src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs b/src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs deleted file mode 100644 index fba62e40aea..00000000000 --- a/src/DynamoCoreWpf/Utilities/LuceneSearchUtility.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System.Linq; -using Dynamo.Configuration; -using Dynamo.Models; -using Lucene.Net.Index; -using Lucene.Net.QueryParsers.Classic; -using Lucene.Net.Search; - -namespace Dynamo.Utilities -{ - internal class LuceneSearchUtility - { - internal DynamoModel dynamoModel; - - internal LuceneSearchUtility(DynamoModel model) - { - dynamoModel = model; - } - - /// - /// Creates a search query with adjusted priority, fuzzy logic and wildcards. - /// Complete Search term appearing in Name of the package will be given highest priority. - /// Then, complete search term appearing in other metadata, - /// Then, a part of the search term(if containing multiple words) appearing in Name of the package - /// Then, a part of the search term appearing in other metadata of the package. - /// Then priority will be given based on fuzzy logic- that is if the complete search term may have been misspelled for upto 2(max edits) characters. - /// Then, the same fuzzy logic will be applied to each part of the search term. - /// - /// All fields to be searched in. - /// Search key to be searched for. - /// - internal string CreateSearchQuery(string[] fields, string SearchTerm) - { - int fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMinEdits; - // Use a larger max edit value - more tolerant with typo when search term is longer than threshold - if (SearchTerm.Length > LuceneConfig.FuzzySearchMaxEditsThreshold) - { - fuzzyLogicMaxEdits = LuceneConfig.FuzzySearchMaxEdits; - } - - var booleanQuery = new BooleanQuery(); - string searchTerm = QueryParser.Escape(SearchTerm); - - foreach (string f in fields) - { - FuzzyQuery fuzzyQuery; - if (searchTerm.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, searchTerm), fuzzyLogicMaxEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - - var wildcardQuery = new WildcardQuery(new Term(f, searchTerm)); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.SearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.SearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - wildcardQuery = new WildcardQuery(new Term(f, "*" + searchTerm + "*")); - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchNameWeight; - } - else - { - wildcardQuery.Boost = LuceneConfig.WildcardsSearchMetaFieldsWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - - if (searchTerm.Contains(' ') || searchTerm.Contains('.')) - { - foreach (string s in searchTerm.Split(' ', '.')) - { - if (s.Length > LuceneConfig.FuzzySearchMinimalTermLength) - { - fuzzyQuery = new FuzzyQuery(new Term(f, s), LuceneConfig.FuzzySearchMinEdits); - booleanQuery.Add(fuzzyQuery, Occur.SHOULD); - } - wildcardQuery = new WildcardQuery(new Term(f, "*" + s + "*")); - - if (f.Equals(nameof(LuceneConfig.IndexFieldsEnum.Name))) - { - wildcardQuery.Boost = 5; - } - else - { - wildcardQuery.Boost = LuceneConfig.FuzzySearchWeight; - } - booleanQuery.Add(wildcardQuery, Occur.SHOULD); - } - } - } - return booleanQuery.ToString(); - } - } -} diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index f067250c629..1e43f3a6180 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -3,7 +3,6 @@ using System.Collections.ObjectModel; using System.Collections.Specialized; using System.ComponentModel; -using System.IO; using System.Linq; using System.Threading.Tasks; using System.Windows; @@ -19,10 +18,7 @@ using Dynamo.Wpf.Properties; using Dynamo.Wpf.Utilities; using Greg.Responses; -using Lucene.Net.Analysis; -using Lucene.Net.Analysis.Standard; using Lucene.Net.Documents; -using Lucene.Net.Index; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; using Microsoft.Practices.Prism.Commands; @@ -145,17 +141,6 @@ private void SetFilterHosts(object obj) } #region Properties & Fields - internal List addedFields; - internal DirectoryReader dirReader; - internal Lucene.Net.Store.Directory indexDir; - internal IndexWriter writer; - - // Holds the instance for the IndexSearcher - internal IndexSearcher Searcher; - - // Used for creating the StandardAnalyzer - internal Analyzer Analyzer; - // Lucene search view model. internal LuceneSearchUtility LuceneSearchUtility { get; set; } @@ -460,57 +445,6 @@ internal PackageManagerSearchViewModel() SelectedHosts = new List(); } - /// - /// Initialize Lucene config file writer. - /// - private void InitializeLuceneConfig() - { - addedFields = new List(); - - var dynamoModel = PackageManagerClientViewModel.DynamoViewModel.Model; - - DirectoryInfo webBrowserUserDataFolder; - var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); - webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; - - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, "Index", LuceneConfig.PackagesIndexingDirectory); - indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); - - // Create an analyzer to process the text - Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); - - // Initialize Lucene index writer, unless in test mode. - if (!DynamoModel.IsTestMode) - { - // Create an index writer - IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) - { - OpenMode = OpenMode.CREATE - }; - writer = new IndexWriter(indexDir, indexConfig); - } - } - - /// - /// Initialize Lucene index document object for reuse - /// - /// - private Document InitializeIndexDocument() - { - if (DynamoModel.IsTestMode) return null; - - var name = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Name), string.Empty, Field.Store.YES); - var description = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Description), string.Empty, Field.Store.YES); - var keywords = new TextField(nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), string.Empty, Field.Store.YES); - var hosts = new TextField(nameof(LuceneConfig.IndexFieldsEnum.Hosts), string.Empty, Field.Store.YES); - - var d = new Document() - { - name, description, keywords, hosts - }; - return d; - } - /// /// Add package information to Lucene index /// @@ -519,52 +453,22 @@ private Document InitializeIndexDocument() private void AddPackageToSearchIndex(PackageManagerSearchElement package, Document doc) { if (DynamoModel.IsTestMode) return; - if (addedFields == null) return; + if (LuceneSearchUtility.addedFields == null) return; - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Name), package.Name); - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Description), package.Description); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Name), package.Name); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Description), package.Description); if (package.Keywords.Count() > 0) { - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), package.Keywords); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.SearchKeywords), package.Keywords); } if (package.Hosts != null && string.IsNullOrEmpty(package.Hosts.ToString())) { - SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Hosts), package.Hosts.ToString(), true, true); + LuceneSearchUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.IndexFieldsEnum.Hosts), package.Hosts.ToString(), true, true); } - writer?.AddDocument(doc); - } - - /// - /// The SetDocumentFieldValue method should be optimized later - /// - /// Lucene document in which the package info is stored - /// Package field that is being updated in the document - /// Package field value - /// This is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. - /// This is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. - private void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) - { - addedFields.Add(field); - if (isTextField && !field.Equals("DocName")) - { - ((TextField)doc.GetField(field)).SetStringValue(value); - } - else - { - ((StringField)doc.GetField(field)).SetStringValue(value); - } - if (isLast) - { - List diff = LuceneConfig.PackageIndexFields.Except(addedFields).ToList(); - foreach (var d in diff) - { - SetDocumentFieldValue(doc, d, ""); - } - addedFields.Clear(); - } + LuceneSearchUtility.writer?.AddDocument(doc); } /// @@ -575,7 +479,7 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi PackageManagerClientViewModel = client; HostFilter = InitializeHostFilter(); LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model); - InitializeLuceneConfig(); + LuceneSearchUtility.InitializeLuceneConfig(LuceneConfig.PackagesIndexingDirectory); } /// @@ -854,7 +758,7 @@ public void RefreshAndSearchAsync() this.ClearSearchResults(); this.SearchState = PackageSearchState.Syncing; - var iDoc = InitializeIndexDocument(); + var iDoc = LuceneSearchUtility.InitializeIndexDocumentForPackages(); Task>.Factory.StartNew(RefreshAndSearch).ContinueWith((t) => { @@ -873,13 +777,13 @@ public void RefreshAndSearchAsync() if (!DynamoModel.IsTestMode) { - dirReader = writer?.GetReader(applyAllDeletes: true); - Searcher = new IndexSearcher(dirReader); + LuceneSearchUtility.dirReader = LuceneSearchUtility.writer?.GetReader(applyAllDeletes: true); + LuceneSearchUtility.Searcher = new IndexSearcher(LuceneSearchUtility.dirReader); - writer?.Commit(); - writer?.Dispose(); - indexDir?.Dispose(); - writer = null; + LuceneSearchUtility.writer?.Commit(); + LuceneSearchUtility.writer?.Dispose(); + LuceneSearchUtility.indexDir?.Dispose(); + LuceneSearchUtility.writer = null; } } RefreshInfectedPackages(); @@ -1211,23 +1115,21 @@ internal IEnumerable Search(string searchT string searchTerm = searchText.Trim(); var packages = new List(); - string[] fnames = LuceneConfig.PackageIndexFields; - - var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, fnames, Analyzer) + var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.PackageIndexFields, LuceneSearchUtility.Analyzer) { AllowLeadingWildcard = true, DefaultOperator = LuceneConfig.DefaultOperator, FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(fnames, searchTerm)); + Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(LuceneConfig.PackageIndexFields, searchTerm)); //indicate we want the first 50 results - TopDocs topDocs = Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); + TopDocs topDocs = LuceneSearchUtility.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); for (int i = 0; i < topDocs.ScoreDocs.Length; i++) { //read back a doc from results - Document resultDoc = Searcher.Doc(topDocs.ScoreDocs[i].Doc); + Document resultDoc = LuceneSearchUtility.Searcher.Doc(topDocs.ScoreDocs[i].Doc); // Get the view model of the package element and add it to the results. string name = resultDoc.Get(nameof(LuceneConfig.IndexFieldsEnum.Name)); diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index aeeab593e90..0d01e4b4335 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -372,7 +372,7 @@ internal SearchViewModel(DynamoViewModel dynamoViewModel) iconServices = new IconServices(pathManager); InitializeCore(); - LuceneSearchUtility = new LuceneSearchUtility(dynamoViewModel.Model); + LuceneSearchUtility = dynamoViewModel.Model.LuceneSearchUtility; } // Just for tests. Please refer to LibraryTests.cs @@ -941,20 +941,20 @@ internal IEnumerable Search(string search, bool useL { string searchTerm = search.Trim(); var candidates = new List(); - var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.IndexFields, Model.Analyzer) + var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtility.Analyzer) { AllowLeadingWildcard = true, DefaultOperator = LuceneConfig.DefaultOperator, FuzzyMinSim = LuceneConfig.MinimumSimilarity }; - Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(LuceneConfig.IndexFields, searchTerm)); - TopDocs topDocs = Model.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); + Query query = parser.Parse(LuceneSearchUtility.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm)); + TopDocs topDocs = LuceneSearchUtility.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); for (int i = 0; i < topDocs.ScoreDocs.Length; i++) { // read back a Lucene doc from results - Document resultDoc = Model.Searcher.Doc(topDocs.ScoreDocs[i].Doc); + Document resultDoc = LuceneSearchUtility.Searcher.Doc(topDocs.ScoreDocs[i].Doc); string name = resultDoc.Get(nameof(LuceneConfig.IndexFieldsEnum.Name)); string docName = resultDoc.Get(nameof(LuceneConfig.IndexFieldsEnum.DocName)); From f877bae005ad3d250484bf963440c52313fba419 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:38:48 -0400 Subject: [PATCH 07/10] Update LuceneSearchUtility.cs --- src/DynamoCore/Utilities/LuceneSearchUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index f09f2cff682..f34747fc17e 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -119,7 +119,7 @@ internal Document InitializeIndexDocumentForPackages() /// Field that is being updated in the document /// Field value /// This is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. - /// This is used for the last value set in the document, and it will fetch all the other field not set for the document and add them with an empty string. + /// This is used for the last value set in the document. It will fetch all the other field not set for the document and add them with an empty string. internal void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) { string[] indexedFields = null; From 2dbdafe2940bc8a6ecd67c0e137f65dff43260e0 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:40:11 -0400 Subject: [PATCH 08/10] Update LuceneSearchUtility.cs --- src/DynamoCore/Utilities/LuceneSearchUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index f34747fc17e..c2c7c07525c 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -119,7 +119,7 @@ internal Document InitializeIndexDocumentForPackages() /// Field that is being updated in the document /// Field value /// This is used when the value need to be tokenized(broken down into pieces), whereas StringTextFields are tokenized. - /// This is used for the last value set in the document. It will fetch all the other field not set for the document and add them with an empty string. + /// This is used for the last value set in the document. It will fetch all the fields not set in the document and add them with an empty string. internal void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) { string[] indexedFields = null; From 8418174396cb7d30be8a337a03ad189429c85e59 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:05:17 -0400 Subject: [PATCH 09/10] comments --- .../ViewModels/PackageManager/PackageManagerSearchViewModel.cs | 2 +- src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index 1e43f3a6180..ed9bb69bb31 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -141,7 +141,7 @@ private void SetFilterHosts(object obj) } #region Properties & Fields - // Lucene search view model. + // Lucene search utility to perform indexing operations. internal LuceneSearchUtility LuceneSearchUtility { get; set; } // The results of the last synchronization with the package manager server diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index 0d01e4b4335..3a49f6e1c93 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -349,7 +349,7 @@ public NodeSearchElementViewModel FindViewModelForNode(string nodeName) public NodeSearchModel Model { get; private set; } internal readonly DynamoViewModel dynamoViewModel; - // Lucene search view model. + // Lucene search utility to perform indexing operations. internal LuceneSearchUtility LuceneSearchUtility { get; set; } /// From af4e984d93acbb05af0cf48e659879782091fff0 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:38:39 -0400 Subject: [PATCH 10/10] Update SearchViewModel.cs --- src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index 3a49f6e1c93..9977a344390 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -350,7 +350,7 @@ public NodeSearchElementViewModel FindViewModelForNode(string nodeName) internal readonly DynamoViewModel dynamoViewModel; // Lucene search utility to perform indexing operations. - internal LuceneSearchUtility LuceneSearchUtility { get; set; } + internal LuceneSearchUtility LuceneSearchUtility { get; } /// /// Class name, that has been clicked in library search view.