Lucene index amend improvements

src/DynamoCore/Search/NodeSearchModel.cs

@@ -236,17 +236,7 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
 
             if (luceneSearchUtility != null)
             {
-                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
-                if (luceneSearchUtility.writer != null)
-                {
-                    luceneSearchUtility.dirReader = luceneSearchUtility.writer.GetReader(applyAllDeletes: true);
-                }
-                else
-                {
-                    luceneSearchUtility.dirReader = DirectoryReader.Open(luceneSearchUtility.indexDir);
-                }
-                luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader);
-
+                luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.DirReader);
                 string searchTerm = search.Trim();
                 var candidates = new List<NodeSearchElement>();
                 var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, luceneSearchUtility.Analyzer)

src/DynamoCore/Utilities/LuceneSearchUtility.cs

@@ -3,12 +3,8 @@
 using System.IO;
 using System.Linq;
 using Dynamo.Configuration;
-using Dynamo.Core;
-using Dynamo.Events;
-using Dynamo.Logging;
 using Dynamo.Models;
 using Dynamo.Search.SearchElements;
-using Dynamo.Session;
 using Lucene.Net.Analysis;
 using Lucene.Net.Analysis.Br;
 using Lucene.Net.Analysis.Cjk;
@@ -45,15 +41,22 @@ internal class LuceneSearchUtility
         /// <summary>
         /// Lucene Directory Reader
         /// </summary>
-        internal DirectoryReader dirReader;
+        internal DirectoryReader DirReader
+        {
+            get
+            {
+                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
+                return writer != null ? writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(indexDir);
+            }
+        }
 
         /// <summary>
         /// Lucene Index Directory, it can be RAMDirectory or FSDirectory
         /// </summary>
         internal Lucene.Net.Store.Directory indexDir;
 
         /// <summary>
-        /// Lucene Index write
+        /// Lucene Index writer
         /// </summary>
         internal IndexWriter writer;
 
@@ -95,6 +98,7 @@ public enum LuceneStorage
         /// <summary>
         /// Constructor for LuceneSearchUtility, it will use the storage type passed as parameter
         /// </summary>
+        /// <param name="model"></param>
         /// <param name="config"></param>
         internal LuceneSearchUtility(DynamoModel model, LuceneStartConfig config)
         {
@@ -132,12 +136,13 @@ internal void InitializeLuceneConfig()
         /// <summary>
         /// Create index writer for followup doc indexing
         /// </summary>
-        internal void CreateLuceneIndexWriter()
+        /// <param name="mode">Index open mode for Lucene index writer</param>
+        internal void CreateLuceneIndexWriter(OpenMode mode = OpenMode.CREATE)
         {
             // Create an index writer
             IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer)
             {
-                OpenMode = OpenMode.CREATE
+                OpenMode = mode
             };
             try
             {
@@ -147,12 +152,12 @@ internal void CreateLuceneIndexWriter()
             {
 
                 DisposeWriter();
-                (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET LockObtainFailedException {ex}");
+                dynamoModel.Logger.LogError($"LuceneNET LockObtainFailedException {ex}");
 
             }
             catch (Exception ex)
             {
-                (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET Exception {ex}");
+                dynamoModel.Logger.LogError($"LuceneNET Exception {ex}");
             }
         }
 
@@ -384,6 +389,9 @@ internal Analyzer CreateAnalyzerByLanguage(string language)
             }
         }
 
+        /// <summary>
+        /// Dispose Lucene index write objects and reuse other objects
+        /// </summary>
         internal void DisposeWriter()
         {
             writer?.Dispose();
@@ -396,7 +404,7 @@ internal void DisposeWriter()
         internal void DisposeAll()
         {
             writer?.Dispose();
-            dirReader?.Dispose();
+            DirReader?.Dispose();
             indexDir?.Dispose();
             Analyzer?.Dispose();
         }
@@ -418,6 +426,16 @@ internal void CommitWriterChanges()
         internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
         {
             if (addedFields == null) return;
+            // During DynamoModel initialization, the index writer should still be valid here
+            // If the index writer is null and index not locked, it means the index writer has been disposed, e.g. DynamoModel finished initialization
+            // If the index writer is null and index locked, it means another Dynamo session is currently updating the search index
+            // Try to create a new index writer to amend the index
+            if (writer == null && !IndexWriter.IsLocked(this.indexDir))
+            {
+                CreateLuceneIndexWriter(OpenMode.CREATE_OR_APPEND);
+            }
+            // If the index writer is still null, skip the indexing
+            if (writer == null) return;
 
             SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
             SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);

src/DynamoCoreWpf/Controls/StartPage.xaml.cs

@@ -4,6 +4,7 @@
 using System.Collections.Specialized;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Input;
@@ -366,7 +367,7 @@ private void RefreshFileList(ObservableCollection<StartPageListItem> files,
             IEnumerable<string> filePaths)
         {
             files.Clear();
-            foreach (var filePath in filePaths)
+            foreach (var filePath in filePaths.Where(x => x != null))
             {
                 try
                 {

src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs

@@ -1,14 +1,3 @@
-using Dynamo.Configuration;
-using Dynamo.Engine.CodeGeneration;
-using Dynamo.Graph;
-using Dynamo.Graph.Nodes;
-using Dynamo.Graph.Nodes.CustomNodes;
-using Dynamo.Graph.Workspaces;
-using Dynamo.Logging;
-using Dynamo.Models;
-using Dynamo.Selection;
-using Dynamo.Wpf.ViewModels.Core;
-using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
@@ -20,6 +9,17 @@
 using System.Windows;
 using System.Windows.Controls.Primitives;
 using System.Windows.Media;
+using Dynamo.Configuration;
+using Dynamo.Engine.CodeGeneration;
+using Dynamo.Graph;
+using Dynamo.Graph.Nodes;
+using Dynamo.Graph.Nodes.CustomNodes;
+using Dynamo.Graph.Workspaces;
+using Dynamo.Logging;
+using Dynamo.Models;
+using Dynamo.Selection;
+using Dynamo.Wpf.ViewModels.Core;
+using Newtonsoft.Json;
 using Point = System.Windows.Point;
 using Size = System.Windows.Size;
 

src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs

@@ -1010,6 +1010,8 @@ internal virtual void InstallPackage(PackageDownloadHandle packageDownloadHandle
                         }
                     }
                     SetPackageState(packageDownloadHandle, installPath);
+                    // Dispose Index writer to avoid file lock after new package is installed
+                    Search.LuceneSearch.LuceneUtilityNodeSearch.DisposeWriter();
                     Analytics.TrackEvent(Actions.Installed, Categories.PackageManagerOperations, $"{packageDownloadHandle?.Name}");
                 }
                 catch (Exception e)

src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs

@@ -1028,16 +1028,7 @@ public void RefreshAndSearchAsync()
 
                     if (!DynamoModel.IsTestMode)
                     {
-                        if (LuceneUtility.writer != null)
-                        {
-                            LuceneUtility.dirReader = LuceneUtility.writer.GetReader(applyAllDeletes: true);
-                        }
-                        else
-                        {
-                            LuceneUtility.dirReader = DirectoryReader.Open(LuceneUtility.indexDir);
-                        }
-                        LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.dirReader);
-
+                        LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.DirReader);
                         LuceneUtility.CommitWriterChanges();
                         LuceneUtility.DisposeWriter();
                     }
@@ -1416,14 +1407,7 @@ internal IEnumerable<PackageManagerSearchElementViewModel> Search(string searchT
                 string searchTerm = searchText.Trim();
                 var packages = new List<PackageManagerSearchElementViewModel>();
 
-                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method,otherwise new indexed info won't be reflected
-                LuceneUtility.dirReader = LuceneUtility.writer?.GetReader(applyAllDeletes: true);
-
-                if (LuceneUtility.Searcher == null && LuceneUtility.dirReader != null)
-                {
-                    LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.dirReader);
-                }
-
+                LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.DirReader);
                 var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.PackageIndexFields, LuceneUtility.Analyzer)
                 {
                     AllowLeadingWildcard = true,
@@ -1577,7 +1561,6 @@ internal void Close()
             InitialResultsLoaded = false;   // reset the loading screen settings
             RequestShowFileDialog -= OnRequestShowFileDialog;
             nonHostFilter.ForEach(f => f.PropertyChanged -= filter_PropertyChanged);
-            LuceneUtility.DisposeAll();
         }
     }
 }

src/DynamoCoreWpf/ViewModels/PackageManager/PackagePathViewModel.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using Dynamo.Core;
@@ -266,6 +266,8 @@ private void CommitChanges(object param)
                 packageLoader.LoadNewCustomNodesAndPackages(newPaths, customNodeManager);
             }
             packagePathsEnabled.Clear();
+            // Dispose node Index writer to avoid file lock after new package path applied and packages loaded.
+            Search.LuceneSearch.LuceneUtilityNodeSearch.DisposeWriter();
         }
 
         internal void SetPackagesScheduledState(string packagePath, bool packagePathDisabled)

src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs

@@ -621,11 +621,7 @@ internal IEnumerable<NodeSearchElementViewModel> SearchNodeAutocomplete(string s
         {
             if (LuceneUtility != null)
             {
-                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
-                LuceneUtility.dirReader = LuceneUtility.writer?.GetReader(applyAllDeletes: true);
-                if (LuceneUtility.dirReader == null) return null;
-
-                LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.dirReader);
+                LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.DirReader);
 
                 string searchTerm = search.Trim();
                 var candidates = new List<NodeSearchElementViewModel>();

src/DynamoUtilities/PathHelper.cs

@@ -37,6 +37,9 @@ public static Exception CreateFolderIfNotExist(string folderPath)
         {
             try
             {
+                // Do not even try when folder path is null or empty.
+                // This usually happens when system folder dialog is initialized with empty path
+                if (string.IsNullOrEmpty(folderPath)) return null;
                 // When network path is access denied, the Directory.Exits however still 
                 // return true.
                 // EnumerateDirectories operation is additional check