Memoize lazily evaluated sequences

Core/Extensions/EnumerableExtensions.cs

@@ -1,10 +1,11 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 
 namespace CKAN.Extensions
 {
-    internal static class EnumerableExtensions
+    public static class EnumerableExtensions
     {
         public static ICollection<T> AsCollection<T>(this IEnumerable<T> source)
         {
@@ -21,5 +22,100 @@ public static HashSet<T> ToHashSet<T>(this IEnumerable<T> source)
 
             return new HashSet<T>(source);
         }
+
+        public static IEnumerable<T> Memoize<T>(this IEnumerable<T> source)
+        {
+            if (source == null)
+            {
+                throw new ArgumentNullException("source");
+            }
+            else if (source is Memoized<T>)
+            {
+                // Already memoized, don't wrap another layer
+                return source;
+            }
+            else
+            {
+                return new Memoized<T>(source);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Memoized lazy evaluation in C#!
+    /// From https://stackoverflow.com/a/12428250/2422988
+    /// </summary>
+    public class Memoized<T> : IEnumerable<T>
+    {
+        public Memoized(IEnumerable<T> source)
+        {
+            this.source = source;
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            lock (gate)
+            {
+                if (isCacheComplete)
+                {
+                    return cache.GetEnumerator();
+                }
+                else if (enumerator == null)
+                {
+                    enumerator = source.GetEnumerator();
+                }
+            }
+            return GetMemoizingEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        private IEnumerator<T> GetMemoizingEnumerator()
+        {
+            for (Int32 index = 0; TryGetItem(index, out T item); ++index)
+            {
+                yield return item;
+            }
+        }
+
+        private bool TryGetItem(Int32 index, out T item)
+        {
+            lock (gate)
+            {
+                if (!IsItemInCache(index))
+                {
+                    // The iteration may have completed while waiting for the lock
+                    if (isCacheComplete)
+                    {
+                        item = default(T);
+                        return false;
+                    }
+                    if (!enumerator.MoveNext())
+                    {
+                        item = default(T);
+                        isCacheComplete = true;
+                        enumerator.Dispose();
+                        return false;
+                    }
+                    cache.Add(enumerator.Current);
+                }
+                item = cache[index];
+                return true;
+            }
+        }
+
+        private bool IsItemInCache(Int32 index)
+        {
+            return index < cache.Count;
+        }
+
+        private readonly IEnumerable<T> source;
+        private          IEnumerator<T> enumerator;
+        private readonly List<T>        cache = new List<T>();
+        private          bool           isCacheComplete;
+        private readonly object         gate = new object();
     }
 }

Core/ModuleInstaller.cs

@@ -4,14 +4,14 @@
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Transactions;
-using CKAN.Extensions;
 using ICSharpCode.SharpZipLib.Core;
 using ICSharpCode.SharpZipLib.Zip;
 using log4net;
-using CKAN.Versioning;
 using ChinhDo.Transactions.FileManager;
-using CKAN.Configuration;
 using Autofac;
+using CKAN.Extensions;
+using CKAN.Versioning;
+using CKAN.Configuration;
 
 namespace CKAN
 {
@@ -728,6 +728,7 @@ internal static void CopyZipEntry(ZipFile zipfile, ZipEntry entry, string fullPa
         /// </summary>
         public void UninstallList(IEnumerable<string> mods, bool ConfirmPrompt = true, IEnumerable<string> installing = null)
         {
+            mods = mods.Memoize();
             // Pre-check, have they even asked for things which are installed?
 
             foreach (string mod in mods.Where(mod => registry_manager.registry.InstalledModule(mod) == null))
@@ -736,13 +737,15 @@ public void UninstallList(IEnumerable<string> mods, bool ConfirmPrompt = true, I
             }
 
             // Find all the things which need uninstalling.
-            IEnumerable<string> revdep = mods.Union(
-                registry_manager.registry.FindReverseDependencies(
-                    mods.Except(installing ?? new string[] {})));
+            IEnumerable<string> revdep = mods
+                .Union(registry_manager.registry.FindReverseDependencies(
+                    mods.Except(installing ?? new string[] {})))
+                .Memoize();
             IEnumerable<string> goners = revdep.Union(
-                registry_manager.registry.FindRemovableAutoInstalled(
-                    registry_manager.registry.InstalledModules.Where(im => !revdep.Contains(im.identifier))
-                ).Select(im => im.identifier));
+                    registry_manager.registry.FindRemovableAutoInstalled(
+                        registry_manager.registry.InstalledModules.Where(im => !revdep.Contains(im.identifier)))
+                    .Select(im => im.identifier))
+                .Memoize();
 
             // If there us nothing to uninstall, skip out.
             if (!goners.Any())
@@ -984,6 +987,8 @@ public void AddRemove(IEnumerable<CkanModule> add = null, IEnumerable<InstalledM
 
             using (var tx = CkanTransaction.CreateTransactionScope())
             {
+                remove = remove.Memoize();
+                add    = add.Memoize();
                 int totSteps = (remove?.Count() ?? 0)
                              + (add?.Count()    ?? 0);
                 int step = 0;
@@ -1031,6 +1036,7 @@ public void Upgrade(IEnumerable<string> identifiers, IDownloader netAsyncDownloa
         /// </summary>
         public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloader, bool enforceConsistency = true)
         {
+            modules = modules.Memoize();
             User.RaiseMessage("About to upgrade...\r\n");
 
             // Start by making sure we've downloaded everything.
@@ -1094,6 +1100,7 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
         /// </summary>
         public void Replace(IEnumerable<ModuleReplacement> replacements, RelationshipResolverOptions options, IDownloader netAsyncDownloader, bool enforceConsistency = true)
         {
+            replacements = replacements.Memoize();
             log.Debug("Using Replace method");
             List<CkanModule> modsToInstall = new List<CkanModule>();
             var modsToRemove = new List<InstalledModule>();

Core/Registry/AvailableModule.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Runtime.Serialization;
+using CKAN.Extensions;
 using CKAN.Versioning;
 using log4net;
 using Newtonsoft.Json;
@@ -87,7 +88,7 @@ public CkanModule Latest(
         )
         {
             log.DebugFormat("Our dictionary has {0} keys", module_version.Keys.Count);
-            IEnumerable<CkanModule> modules = module_version.Values;
+            IEnumerable<CkanModule> modules = module_version.Values.Reverse();
             if (relationship != null)
             {
                 modules = modules.Where(relationship.WithinBounds);
@@ -104,11 +105,12 @@ public CkanModule Latest(
             {
                 modules = modules.Where(m => DependsAndConflictsOK(m, toInstall));
             }
-            return modules.LastOrDefault();
+            return modules.FirstOrDefault();
         }
 
         private static bool DependsAndConflictsOK(CkanModule module, IEnumerable<CkanModule> others)
         {
+            others = others.Memoize();
             if (module.depends != null)
             {
                 foreach (RelationshipDescriptor rel in module.depends)
@@ -123,7 +125,7 @@ private static bool DependsAndConflictsOK(CkanModule module, IEnumerable<CkanMod
             if (module.conflicts != null)
             {
                 // Skip self-conflicts (but catch other modules providing self)
-                var othersMinusSelf = others.Where(m => m.identifier != module.identifier);
+                var othersMinusSelf = others.Where(m => m.identifier != module.identifier).Memoize();
                 foreach (RelationshipDescriptor rel in module.conflicts)
                 {
                     // If any of the conflicts are present, fail

Core/Registry/Registry.cs

@@ -746,8 +746,8 @@ public CkanModule GetModuleByVersion(string ident, ModuleVersion version)
         }
 
         /// <summary>
-        ///     Register the supplied module as having been installed, thereby keeping
-        ///     track of its metadata and files.
+        /// Register the supplied module as having been installed, thereby keeping
+        /// track of its metadata and files.
         /// </summary>
         public void RegisterModule(CkanModule mod, IEnumerable<string> absolute_files, KSP ksp, bool autoInstalled)
         {
@@ -760,7 +760,9 @@ public void RegisterModule(CkanModule mod, IEnumerable<string> absolute_files, K
             var inconsistencies = new List<string>();
 
             // We always work with relative files, so let's get some!
-            IEnumerable<string> relative_files = absolute_files.Select(x => ksp.ToRelativeGameDir(x));
+            IEnumerable<string> relative_files = absolute_files
+                .Select(x => ksp.ToRelativeGameDir(x))
+                .Memoize();
 
             // For now, it's always cool if a module wants to register a directory.
             // We have to flip back to absolute paths to actually test this.
@@ -771,10 +773,10 @@ public void RegisterModule(CkanModule mod, IEnumerable<string> absolute_files, K
                 {
                     // Woah! Registering an already owned file? Not cool!
                     // (Although if it existed, we should have thrown a kraken well before this.)
-                    inconsistencies.Add(
-                        string.Format("{0} wishes to install {1}, but this file is registered to {2}",
-                            mod.identifier, file, owner
-                            ));
+                    inconsistencies.Add(string.Format(
+                        "{0} wishes to install {1}, but this file is registered to {2}",
+                        mod.identifier, file, owner
+                    ));
                 }
             }
 
@@ -1082,6 +1084,9 @@ internal static IEnumerable<string> FindReverseDependencies(
             IDictionary<string, UnmanagedModuleVersion> dlc
         )
         {
+            modules_to_remove = modules_to_remove.Memoize();
+            orig_installed    = orig_installed.Memoize();
+            var dllSet = dlls.ToHashSet();
             // The empty list has no reverse dependencies
             // (Don't remove broken modules if we're only installing)
             if (modules_to_remove.Any())
@@ -1097,10 +1102,10 @@ IDictionary<string, UnmanagedModuleVersion> dlc
                     HashSet<CkanModule> hypothetical = new HashSet<CkanModule>(orig_installed); // Clone because we alter hypothetical.
                     hypothetical.RemoveWhere(mod => modules_to_remove.Contains(mod.identifier));
 
-                    log.DebugFormat("Started with {0}, removing {1}, and keeping {2}; our dlls are {3}", string.Join(", ", orig_installed), string.Join(", ", modules_to_remove), string.Join(", ", hypothetical), string.Join(", ", dlls));
+                    log.DebugFormat("Started with {0}, removing {1}, and keeping {2}; our dlls are {3}", string.Join(", ", orig_installed), string.Join(", ", modules_to_remove), string.Join(", ", hypothetical), string.Join(", ", dllSet));
 
                     // Find what would break with this configuration.
-                    var broken = SanityChecker.FindUnsatisfiedDepends(hypothetical, dlls.ToHashSet(), dlc)
+                    var broken = SanityChecker.FindUnsatisfiedDepends(hypothetical, dllSet, dlc)
                         .Select(x => x.Key.identifier).ToHashSet();
 
                     // Lazily return each newly found rev dep
@@ -1151,6 +1156,7 @@ IDictionary<string, UnmanagedModuleVersion> dlc
         )
         {
             // ToList ensures that the collection isn't modified while the enumeration operation is executing
+            installedModules = installedModules.Memoize();
             var autoInstMods = installedModules.Where(im => im.AutoInstalled).ToList();
             var autoInstIds  = autoInstMods.Select(im => im.Module.identifier).ToHashSet();
             var instCkanMods = installedModules.Select(im => im.Module);

Core/Relationships/RelationshipResolver.cs

@@ -311,6 +311,8 @@ private void Resolve(CkanModule module, RelationshipResolverOptions options, IEn
             // is true.
             var sub_options = (RelationshipResolverOptions) options.Clone();
             sub_options.with_suggests = false;
+
+            old_stanza = old_stanza?.Memoize();
 
             log.DebugFormat("Resolving dependencies for {0}", module.identifier);
             ResolveStanza(module.depends, new SelectionReason.Depends(module), sub_options, false, old_stanza);
@@ -348,6 +350,7 @@ private void ResolveStanza(IEnumerable<RelationshipDescriptor> stanza, Selection
             {
                 return;
             }
+            stanza = stanza.Memoize();
 
             foreach (RelationshipDescriptor descriptor in stanza)
             {

Core/Relationships/SanityChecker.cs

@@ -82,8 +82,10 @@ private static bool CheckConsistency(
             out List<KeyValuePair<CkanModule, RelationshipDescriptor>> Conflicts
         )
         {
-            UnmetDepends = FindUnsatisfiedDepends(modules?.ToList(), dlls?.ToHashSet(), dlc);
-            Conflicts    = FindConflicting(       modules,           dlls?.ToHashSet(), dlc);
+            modules = modules?.Memoize();
+            var dllSet = dlls?.ToHashSet();
+            UnmetDepends = FindUnsatisfiedDepends(modules?.ToList(), dllSet, dlc);
+            Conflicts    = FindConflicting(       modules,           dllSet, dlc);
             return !UnmetDepends.Any() && !Conflicts.Any();
         }
 
@@ -106,6 +108,7 @@ IDictionary<string, UnmanagedModuleVersion> dlc
             var unsat = new List<KeyValuePair<CkanModule, RelationshipDescriptor>>();
             if (modules != null)
             {
+                modules = modules.Memoize();
                 foreach (CkanModule m in modules.Where(m => m.depends != null))
                 {
                     foreach (RelationshipDescriptor dep in m.depends)
@@ -139,11 +142,12 @@ IDictionary<string, UnmanagedModuleVersion> dlc
             var confl = new List<KeyValuePair<CkanModule, RelationshipDescriptor>>();
             if (modules != null)
             {
+                modules = modules.Memoize();
                 foreach (CkanModule m in modules.Where(m => m.conflicts != null))
                 {
                     // Remove self from the list, so we're only comparing to OTHER modules.
                     // Also remove other versions of self, to avoid conflicts during upgrades.
-                    var others = modules.Where(other => other.identifier != m.identifier);
+                    var others = modules.Where(other => other.identifier != m.identifier).Memoize();
                     foreach (RelationshipDescriptor dep in m.conflicts)
                     {
                         if (dep.MatchesAny(others, dlls, dlc))

GUI/GUIMod.cs

@@ -196,9 +196,6 @@ public GUIMod(CkanModule mod, IRegistryQuerier registry, KspVersionCriteria curr
         public GUIMod(string identifier, IRegistryQuerier registry, KspVersionCriteria current_ksp_version, bool? incompatible = null)
         {
             Identifier     = identifier;
-            IsIncompatible = incompatible
-                ?? registry.AllAvailable(identifier)
-                    .All(m => !m.IsCompatibleKSP(current_ksp_version));
             IsAutodetected = registry.IsAutodetected(identifier);
             DownloadCount  = registry.DownloadCount(identifier);
             if (registry.IsAutodetected(identifier))
@@ -216,6 +213,8 @@ public GUIMod(string identifier, IRegistryQuerier registry, KspVersionCriteria c
             {
             }
 
+            IsIncompatible = incompatible ?? LatestCompatibleMod == null;
+
             // Let's try to find the compatibility for this mod. If it's not in the registry at
             // all (because it's a DarkKAN mod) then this might fail.
 

GUI/MainChangeset.cs

@@ -4,6 +4,7 @@
 using System.Drawing;
 using System.ComponentModel;
 using System.Windows.Forms;
+using CKAN.Extensions;
 
 namespace CKAN
 {
@@ -93,8 +94,11 @@ private void ClearChangeSet()
         /// </summary>
         /// <param name="changes">Every leftover ModChange that should be sorted</param>
         /// <param name="parent"></param>
-        private void CreateSortedModList(IEnumerable<ModChange> changes, ModChange parent=null)
+        private void CreateSortedModList(IEnumerable<ModChange> changes, ModChange parent = null)
         {
+            var notUserReq = changes
+                .Where(c => !(c.Reason is SelectionReason.UserRequested))
+                .Memoize();
             foreach (ModChange change in changes)
             {
                 bool goDeeper = parent == null || change.Reason.Parent.identifier == parent.Mod.identifier;
@@ -103,7 +107,7 @@ private void CreateSortedModList(IEnumerable<ModChange> changes, ModChange paren
                 {
                     if (!changeSet.Any(c => c.Mod.identifier == change.Mod.identifier && c.ChangeType != GUIModChangeType.Remove))
                         changeSet.Add(change);
-                    CreateSortedModList(changes.Where(c => !(c.Reason is SelectionReason.UserRequested)), change);
+                    CreateSortedModList(notUserReq, change);
                 }
             }
         }