Caching

Update README.md

Update Installation.md

Desktop install docs.

Z Bulkextensions, and Memory Caching

Cache eviction

Caching

Set eviction policy

Actually use the actual cache. D'oh

Refactoring

Licence

Settings

Fix settings bug.

Tidying

First cut of Search rework #255

More filters

Filter UI Rework nest steps. #255

Layout

Tidying

Untagged images filter

Untagged hint

SQL Fixes

Props

Filter hints

UI

Search UI Rework 99% complete.

Fix unidentified faces search

Group image object tags

DistinctBy

Searchbar work

Mousover

Z-order sorting

Z-order sorting. 2nd attempt...

Fix people mapping

Searchbar layout

Patch

Push fix

Discard instead of await

Replace FTS generation with triggers

Cache eviction fix

Update deps

Bump version

Preload more stuff during search

Update SearchService.cs

Cache

Set cache size

Cache ordering

Damselfly.Core.DbModels/DBAbstractions/BaseModel.cs

@@ -306,14 +306,6 @@ public int SaveChanges(string contextDesc)
             return SaveChangesAsync(contextDesc).GetAwaiter().GetResult();
         }
 
-        public async Task GenFullText( bool first )
-        {
-            if (ReadOnly)
-                return;
-
-            await Task.Run(() => DatabaseSpecialisation.GenFullText(first));
-        }
-
         // TODO - this is Sqlite specific and should move down into the MySqlite provider.
         public async Task<IQueryable<T>> ImageSearch<T>(DbSet<T> resultSet, string query, bool includeAITags) where T:class
         {

Damselfly.Core.DbModels/DBAbstractions/MySqlModel.cs

@@ -165,12 +165,6 @@ public IQueryable<T> ImageSearch<T>(DbSet<T> resultSet, string query, bool Inclu
             throw new NotImplementedException();
         }
 
-        public async Task GenFullText(bool first)
-        {
-            // TODO: What do we do here? Maybe something with LIKE?
-            throw new NotImplementedException();
-        }
-
         public void CreateIndexes(ModelBuilder builder)
         {
         }

Damselfly.Core.DbModels/Interfaces/IDataBase.cs

@@ -24,7 +24,6 @@ public interface IDataBase
         Task<int> BatchDelete<T>(IQueryable<T> query) where T : class;
 
         IQueryable<T> ImageSearch<T>(DbSet<T> resultSet, string query, bool includeAITags) where T : class;
-        Task GenFullText(bool first);
         void CreateIndexes(ModelBuilder builder);
     }
 }

Damselfly.Core.DbModels/efmigrations.md

@@ -6,7 +6,6 @@ EF Migrations
 2. Run this from the root Damselfly folder
 
     dotnet ef migrations add <migrationName> --project Damselfly.Migrations.Sqlite --startup-project Damselfly.Web
-    dotnet ef migrations add AddDataProtectionKeys --project Damselfly.Migrations.Sqlite --startup-project Damselfly.Web
 
 3. Comment the Postgres line and uncomment the Sqlite line in OnConfiguring.
 

Damselfly.Core.Utils/ML/MLUtils.cs

@@ -6,33 +6,36 @@ namespace Damselfly.Core.Utils.ML
 {
     public static class MLUtils
     {
-        public static DirectoryInfo? GetModelFolder()
+        public static DirectoryInfo? ModelFolder
         {
-            var modelFolder = Path.Combine(".", "Models");
-
-            if (!Directory.Exists(modelFolder))
+            get
             {
-                var asm = Assembly.GetExecutingAssembly();
+                var modelFolder = Path.Combine(".", "Models");
 
-                if (asm != null)
+                if (!Directory.Exists(modelFolder))
                 {
-                    var thisAsm = new FileInfo(asm.Location);
+                    var asm = Assembly.GetExecutingAssembly();
 
-                    if (thisAsm != null && thisAsm.Directory != null)
+                    if (asm != null)
                     {
-                        modelFolder = Path.Combine(thisAsm.Directory.FullName, "Models");
+                        var thisAsm = new FileInfo(asm.Location);
+
+                        if (thisAsm != null && thisAsm.Directory != null)
+                        {
+                            modelFolder = Path.Combine(thisAsm.Directory.FullName, "Models");
+                        }
                     }
                 }
-            }
 
-            if (Directory.Exists(modelFolder))
-            {
-                return new DirectoryInfo(modelFolder);
-            }
+                if (Directory.Exists(modelFolder))
+                {
+                    return new DirectoryInfo(modelFolder);
+                }
 
-            Logging.LogError($"Models folder not found: {modelFolder}");
+                Logging.LogError($"Models folder not found: {modelFolder}");
 
-            return null;
+                return null;
+            }
         }
     }
 }

Damselfly.Core.Utils/Utils/EnumerableExtensions.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Collections.Generic;
 
 namespace Damselfly.Core.Utils
@@ -10,5 +11,12 @@ public static bool ArePermutations( this IEnumerable<string> collection1, IEnume
             var set1 = new HashSet<string>(collection1, StringComparer.OrdinalIgnoreCase);
             return set1.SetEquals(collection2);
         }
+
+        public static List<T> GetEnumList<T>()
+        {
+            var enumArray = (T[])Enum.GetValues(typeof(T));
+
+            return enumArray.ToList();
+        }
     }
 }

Damselfly.Core.Utils/Utils/ObjectCopier.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Linq;
+
+namespace Damselfly.Core.Utils
+{
+    /// <summary>
+    /// https://stackoverflow.com/questions/3445784/copy-the-property-values-to-another-object-with-c-sharp
+    /// </summary>
+    public static class ObjectUtils
+    {
+        public static bool CopyPropertiesTo<T, TU>(this T source, TU dest)
+        {
+            bool changed = false;
+            var sourceProps = typeof(T).GetProperties().Where(x => x.CanRead).ToList();
+            var destProps = typeof(TU).GetProperties().Where(x => x.CanWrite).ToList();
+
+            foreach (var sourceProp in sourceProps)
+            {
+                var destProp = destProps.FirstOrDefault(x => x.Name == sourceProp.Name);
+
+                if (destProp != null )
+                {
+                    var newVal = sourceProp.GetValue(source, null);
+                    var prevVal = destProp.GetValue(dest, null);
+
+                    if (newVal == null && prevVal == null)
+                        continue;
+
+                    if( newVal == null )
+                    {
+                        destProp.SetValue(dest, null);
+                        Logging.LogVerbose($"Setting property {destProp.Name} to NULL");
+                        changed = true;
+                        continue;
+                    }
+
+                    if( ! newVal.Equals( prevVal ) )
+                    {
+                        Logging.LogVerbose($"Setting property {destProp.Name} to {newVal}");
+                        // check if the property can be set or no.
+                        destProp.SetValue(dest, newVal);
+                        changed = true;
+                    }
+                }
+            }
+
+            return changed;
+        }
+    }
+}
+

Damselfly.Core/Damselfly.Core.csproj

@@ -19,7 +19,7 @@
     <PackageReference Include="MetadataExtractor" Version="2.7.1" />
     <PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
     <PackageReference Include="WordPressPCL" Version="1.9.0" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="1.0.3" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="1.0.4" />
     <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta13" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="font-awesome" Version="4.7.0" />
@@ -30,7 +30,7 @@
     <PackageReference Include="System.Drawing.Common" Version="6.0.0-rc.1.21451.13" />
     <PackageReference Include="Microsoft.AspNetCore.Components.Authorization" Version="6.0.0-rc.1.21452.15" />
     <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.0-rc.1.21452.15" />
-    <PackageReference Include="SendGrid" Version="9.24.2" />
+    <PackageReference Include="SendGrid" Version="9.24.3" />
     <PackageReference Include="MailKit" Version="2.15.0" />
     <PackageReference Include="runtime.osx.10.10-x64.CoreCompat.System.Drawing" Version="5.8.64" />
     <PackageReference Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="6.0.0-rc.1.21452.15" />

Damselfly.Core/ImageProcessing/ImageMagickProcessor.cs

@@ -155,7 +155,7 @@ public void TransformDownloadImage(string input, Stream output, ExportConfig con
             throw new NotImplementedException();
         }
 
-        public async Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo destFile)
+        public Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo destFile)
         {
             throw new NotImplementedException();
         }

Damselfly.Core/Models/ImageContext.cs

@@ -97,6 +97,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             modelBuilder.Entity<Image>().HasIndex(x => x.SortDate);
             modelBuilder.Entity<Folder>().HasIndex(x => x.FolderScanDate);
             modelBuilder.Entity<Folder>().HasIndex(x => x.Path);
+            modelBuilder.Entity<Person>().HasIndex(x => x.State);
             modelBuilder.Entity<Tag>().HasIndex(x => new { x.Keyword }).IsUnique();
 
             modelBuilder.Entity<ImageMetaData>().HasIndex(x => x.ImageId);
@@ -154,7 +155,7 @@ public class Image
 
         // Image File metadata
         public string FileName { get; set; }
-        public ulong FileSizeBytes { get; set; }
+        public int FileSizeBytes { get; set; }
         public DateTime FileCreationDate { get; set; }
         public DateTime FileLastModDate { get; set; }
 
@@ -460,8 +461,7 @@ public class Person
         public enum PersonState
         {
             Unknown = 0,
-            Identified = 1,
-            Confirmed = 2
+            Identified = 1
         };
 
         [Key]
@@ -612,23 +612,42 @@ public enum GroupingType
             Date
         };
 
-        public string SearchText { get; set; }
+        public enum FaceSearchType
+        {
+            None,
+            Faces,
+            NoFaces,
+            IdentifiedFaces,
+            UnidentifiedFaces
+        }
+
+        public enum OrientationType
+        {
+            All,
+            Landscape,
+            Portrait
+        }
+
+        public string SearchText { get; set; } = string.Empty;
         public DateTime? MaxDate { get; set; } = null;
         public DateTime? MinDate { get; set; } = null;
-        public ulong MaxSizeKB { get; set; } = ulong.MaxValue;
-        public ulong MinSizeKB { get; set; } = ulong.MinValue;
-        public Folder Folder { get; set; } = null;
+        public int? MaxSizeKB { get; set; } = null;
+        public int? MinSizeKB { get; set; } = null;
         public bool TagsOnly { get; set; } = false;
         public bool IncludeAITags { get; set; } = true;
+        public bool UntaggedImages { get; set; } = false;
         public int CameraId { get; set; } = -1;
-        public Tag Tag { get; set; } = null;
         public int LensId { get; set; } = -1;
+        public Folder Folder { get; set; } = null;
+        public Tag Tag { get; set; } = null;
         public GroupingType Grouping { get; set; } = GroupingType.None;
         public SortOrderType SortOrder { get; set; } = SortOrderType.Descending;
+        public FaceSearchType FaceSearch { get; set; } = FaceSearchType.None;
+        public OrientationType Orientation { get; set; } = OrientationType.All;
 
         public override string ToString()
         {
-            return $"Filter: T={SearchText}, F={Folder?.FolderId}, Max={MaxDate}, Min={MinDate}, Max={MaxSizeKB}KB, Min={MinSizeKB}KB, Tags={TagsOnly}, Grouping={Grouping}, Sort={SortOrder}";
+            return $"Filter: T={SearchText}, F={Folder?.FolderId}, Max={MaxDate}, Min={MinDate}, Max={MaxSizeKB}KB, Min={MinSizeKB}KB, Tags={TagsOnly}, Grouping={Grouping}, Sort={SortOrder}, Face={FaceSearch}";
         }
     }
 

Damselfly.Core/Models/ScheduledTask.cs

@@ -21,7 +21,8 @@ public enum TaskType
             MetadataScan,
             FlushDBWriteCache,
             DumpPerformance,
-            CleanupThumbs
+            CleanupThumbs,
+            FreeTextIndex
         }
 
         public TaskType Type { get; set; }

Damselfly.Core/ScopedServices/BasketService.cs

@@ -20,6 +20,7 @@ public class BasketService
     {
         private readonly DownloadService _downloadService;
         private readonly UserStatusService _statusService;
+        private readonly ImageCache _imageCache;
 
         private const string s_MyBasket = "My Basket";
         private const string s_DefaultBasket = "default";
@@ -33,9 +34,12 @@ public class BasketService
         public List<Image> BasketImages { get; private set; } = new List<Image>();
 
 
-        public BasketService( UserStatusService statusService, DownloadService downloadService)
+        public BasketService( UserStatusService statusService,
+                                DownloadService downloadService,
+                                ImageCache imageCache)
         {
             _statusService = statusService;
+            _imageCache = imageCache;
             _downloadService = downloadService;
         }
 
@@ -54,26 +58,20 @@ public async Task LoadSelectedImages()
             var watch = new Stopwatch("GetSelectedImages");
 
             // TODO Assign current basket?
-            var images = await db.Baskets.Where( x => x.BasketId == CurrentBasket.BasketId )
+            var images = await db.Baskets.Where(x => x.BasketId == CurrentBasket.BasketId)
                             .Include(x => x.BasketEntries)
-                            .ThenInclude(x => x.Image)
-                            .ThenInclude(x => x.Folder)
-                            .SelectMany( x => x.BasketEntries )
-                            .Select( x => x.Image )
+                            .ThenInclude(b => b.Image)
+                            .SelectMany(x => x.BasketEntries)
+                            .Select(x => x.ImageId)
                             .ToListAsync();
 
-            // We can't used ThenInclude to pull in the image tags due to this
-            // but in the EF framework: https://github.com/dotnet/efcore/issues/19418
-            // It's just too slow. So until they fix it (probably EF 5) we need
-            // to manually explicitly load the tags for each image, which is
-            // very quick.
-            foreach (var img in images)
-                await db.LoadTags(img);
+            // Cache and enrich the entries
+            var enrichedImages = await _imageCache.GetCachedImages(images); 
 
             watch.Stop();
 
             BasketImages.Clear();
-            BasketImages.AddRange(images);
+            BasketImages.AddRange(enrichedImages);
 
             NotifyStateChanged();
         }
@@ -196,7 +194,7 @@ public async Task SetBasketState( ICollection<Image> images, bool newState, Bask
                 var watch = new Stopwatch("SetSelection");
 
                 var existingEntries = db.BasketEntries.Where(x => x.BasketId == basket.BasketId &&
-                            images.Select(img => img.ImageId).Contains(x.ImageId));
+                            images.Select(img => img.ImageId).Contains(x.ImageId)).ToList();
 
                 if (newState)
                 {
@@ -234,8 +232,7 @@ public async Task SetBasketState( ICollection<Image> images, bool newState, Bask
                 }
                 else if (!newState)
                 {
-                    int deleted = await db.BatchDelete( existingEntries );
-                    if( deleted > 0 )
+                    if( await db.BulkDelete( db.BasketEntries, existingEntries ) )
                     {
                         foreach( var image in images )
                         {
@@ -247,7 +244,7 @@ public async Task SetBasketState( ICollection<Image> images, bool newState, Bask
                             BasketImages.RemoveAll(x => images.Select(x => x.ImageId).Contains(x.ImageId));
                             changed = true;
 
-                            _statusService.StatusText = $"Removed {deleted} images from the basket.";
+                            _statusService.StatusText = $"Removed {existingEntries.Count} images from the basket.";
                         }
                     }
                 }