Merge branch 'develop'

.github/workflows/damselfly-actions.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Setup Dotnet Environment
         uses: actions/setup-dotnet@v1
         with:
-          dotnet-version: '6.0.100-rc.1.21458.32'
+          dotnet-version: '6.0.100-rc.2.21505.57'
 
       - name: Run Dotnet Server Build
         run: sh scripts/makeserver.sh ${{ matrix.target }} 

Accord/Accord.Imaging/Accord.Imaging.csproj

@@ -30,7 +30,7 @@
     <None Remove="System.Drawing.Common" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="System.Drawing.Common" Version="6.0.0-rc.1.21451.13" />
+    <PackageReference Include="System.Drawing.Common" Version="6.0.0-rc.2.21480.5" />
   </ItemGroup>
   <ItemGroup>
     <Compile Remove="Properties\AssemblyInfo.cs" />

Damselfly.Core.DbModels/DBAbstractions/BaseModel.cs

@@ -77,7 +77,7 @@ public async Task<bool> BulkInsert<T>(DbSet<T> collection, List<T> itemsToSave)
             if (ReadOnly)
                 return true;
 
-            return await Task.Run(() => DatabaseSpecialisation.BulkInsert(this, collection, itemsToSave));
+            return await DatabaseSpecialisation.BulkInsert(this, collection, itemsToSave);
         }
 
         /// <summary>
@@ -93,7 +93,7 @@ public async Task<bool> BulkUpdate<T>(DbSet<T> collection, List<T> itemsToSave)
             if (ReadOnly)
                 return true;
 
-            return await Task.Run(() => DatabaseSpecialisation.BulkUpdate(this, collection, itemsToSave));
+            return await DatabaseSpecialisation.BulkUpdate(this, collection, itemsToSave);
         }
 
         /// <summary>
@@ -109,7 +109,7 @@ public async Task<bool> BulkDelete<T>(DbSet<T> collection, List<T> itemsToDelete
             if (ReadOnly)
                 return true;
 
-            return await Task.Run( () => DatabaseSpecialisation.BulkDelete(this, collection, itemsToDelete) );
+            return await DatabaseSpecialisation.BulkDelete(this, collection, itemsToDelete);
         }
 
         /// <summary>

Damselfly.Core.DbModels/Damselfly.Core.DbModels.csproj

@@ -1,19 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.0-rc.1.21452.10" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.0-rc.1.21452.10" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="6.0.0-rc.1.21452.10" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.0-rc.1.21452.10">
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.0-rc.2.21480.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.0-rc.2.21480.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="6.0.0-rc.2.21480.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.0-rc.2.21480.5">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0-rc.1.21452.10">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0-rc.2.21480.5">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0-rc.1.21451.13" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.0-rc.1.21452.15" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0-rc.2.21480.5" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.0-rc.2.21480.10" />
   </ItemGroup>
 
   <ItemGroup>

Damselfly.Core.ImageProcessing/Damselfly.Core.ImageProcessing.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Damselfly.Core\Damselfly.Core.csproj" />
+    <ProjectReference Include="..\Damselfly.Core.Interfaces\Damselfly.Core.Interfaces.csproj" />
+    <ProjectReference Include="..\Damselfly.Core.Utils\Damselfly.Core.Utils.csproj" />
+  </ItemGroup>
+</Project>

Damselfly.Core/ImageProcessing/ImageMagickProcessor.cs → Damselfly.Core.ImageProcessing/ImageMagickProcessor.cs

@@ -7,6 +7,7 @@
 using Damselfly.Core.Utils;
 using System.Threading.Tasks;
 using Damselfly.Core.Models;
+using Damselfly.Core.Utils.Images;
 
 namespace Damselfly.Core.ImageProcessing
 {
@@ -15,10 +16,30 @@ public class ImageMagickProcessor : IImageProcessor
         // SkiaSharp doesn't handle .heic files... yet
         private static readonly string[] s_imageExtensions = { ".jpg", ".jpeg", ".png", ".heic", ".tif", ".tiff", ".webp" };
 
-        public static ICollection<string> SupportedFileExtensions { get { return s_imageExtensions; } }
+        public static ICollection<string> SupportedFileExtensions {
+            get {
+                if( imAvailable )
+                    return s_imageExtensions;
+
+                return new string[0];
+            }
+        }
+
+        public string IMVersion
+        {
+            get
+            {
+                if (imAvailable)
+                    return verString;
+
+                return "N/A";
+            }
+        }
 
         const string imageMagickExe = "convert";
         const string graphicsMagickExe = "gm";
+        private static bool imAvailable = false;
+        private string verString = "(not found)";
         private bool s_useGraphicsMagick = false; // GM doesn't support HEIC yet.
 
         public ImageMagickProcessor()
@@ -32,12 +53,22 @@ public ImageMagickProcessor()
         private void CheckToolStatus()
         {
             ProcessStarter improcess = new ProcessStarter();
-            bool imAvailable = improcess.StartProcess("convert", "--version");
+            imAvailable = improcess.StartProcess("convert", "--version");
 
             if (imAvailable)
             {
-                var verString = improcess.OutputText?.Split('\n').FirstOrDefault();
-                Logging.Log($"ImageMagick found: {verString}");
+                var version = improcess.OutputText?.Split('\n').FirstOrDefault() ?? string.Empty;
+
+                if (!string.IsNullOrEmpty(version))
+                {
+                    verString = $"v{version}";
+                    Logging.Log($"ImageMagick found: {verString}");
+                }
+                else
+                {
+                    Logging.LogWarning("No ImageMagick Version returned.");
+                    imAvailable = false;
+                }
             }
             else
                 Logging.LogError("ImageMagick not found.");
@@ -150,7 +181,7 @@ private static void Process_OutputDataReceived(object sender, DataReceivedEventA
                 Logging.LogVerbose(e.Data);
         }
 
-        public void TransformDownloadImage(string input, Stream output, ExportConfig config)
+        public void TransformDownloadImage(string input, Stream output, IExportSettings config)
         {
             throw new NotImplementedException();
         }

Damselfly.Core/ImageProcessing/ImageProcessorFactory.cs → Damselfly.Core.ImageProcessing/ImageProcessorFactory.cs

@@ -3,19 +3,41 @@
 using System.IO;
 using System.Linq;
 using Damselfly.Core.Interfaces;
+using Damselfly.ML.Face.Emgu;
 
 namespace Damselfly.Core.ImageProcessing
 {
-    public class ImageProcessorFactory
+    public class ImageProcessorFactory : IImageProcessorFactory
     {
         private ImageMagickProcessor imProcessor;
         private SkiaSharpProcessor skiaProcessor;
         private ImageSharpProcessor isharpProcessor;
-        private string rootContentPath;
+        private EmguFaceService _emguHashProvider;
+
+        public ImageProcessorFactory( EmguFaceService emguFaceService )
+        {
+            _emguHashProvider = emguFaceService;
+            skiaProcessor = new SkiaSharpProcessor();
+            isharpProcessor = new ImageSharpProcessor();
+            imProcessor = new ImageMagickProcessor();
+        }
 
         public void SetContentPath(string path)
         {
-            rootContentPath = path;
+            isharpProcessor.SetFontPath(Path.Combine(path, "fonts"));
+        }
+
+        /// <summary>
+        /// Get a perceptual hash provider.
+        /// </summary>
+        /// <returns></returns>
+        public IHashProvider GetHashProvider()
+        {
+            return isharpProcessor;
+
+            // EMGU may be faster or better, but needs testing etc.
+            // if (_emguHashProvider.ServiceAvailable)
+            //    return _emguHashProvider;
         }
 
         /// <summary>
@@ -34,29 +56,18 @@ public IImageProcessor GetProcessor( string fileExtension )
             // Skiasharp first. As of 12-Aug-2021, it can do thumbs for 100 images in about 23 seconds
             if (SkiaSharpProcessor.SupportedFileExtensions.Any(x => x.Equals(fileExtension, StringComparison.OrdinalIgnoreCase)))
             {
-                if (skiaProcessor == null)
-                    skiaProcessor = new SkiaSharpProcessor();
-
                 return skiaProcessor;
             }
 
             // ImageSharp next. As of 12-Aug-2021, it can do thumbs for 100 images in about 60 seconds
             if (ImageSharpProcessor.SupportedFileExtensions.Any(x => x.Equals(fileExtension, StringComparison.OrdinalIgnoreCase)))
             {
-                if (isharpProcessor == null)
-                {
-                    isharpProcessor = new ImageSharpProcessor();
-                    isharpProcessor.SetFontPath(Path.Combine(rootContentPath, "fonts"));
-                }
                 return isharpProcessor;
             }
 
             // ImageSharp next. As of 12-Aug-2021, it can do thumbs for 100 images in about 33 seconds. It can also handle HEIC
             if (ImageMagickProcessor.SupportedFileExtensions.Any(x => x.Equals(fileExtension, StringComparison.OrdinalIgnoreCase)))
             {
-                if (imProcessor == null)
-                    imProcessor = new ImageMagickProcessor();
-
                 return imProcessor;
             }
 

Damselfly.Core/ImageProcessing/ImageSharpProcessor.cs → Damselfly.Core.ImageProcessing/ImageSharpProcessor.cs

@@ -4,9 +4,13 @@
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Threading.Tasks;
+using CoenM.ImageHash;
+using CoenM.ImageHash.HashAlgorithms;
 using Damselfly.Core.Interfaces;
 using Damselfly.Core.Models;
 using Damselfly.Core.Utils;
+using Damselfly.Core.Utils.Constants;
+using Damselfly.Core.Utils.Images;
 using SixLabors.Fonts;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.Drawing.Processing;
@@ -18,7 +22,7 @@
 
 namespace Damselfly.Core.ImageProcessing
 {
-    public class ImageSharpProcessor : IImageProcessor
+    public class ImageSharpProcessor : IImageProcessor, IHashProvider
     {
         private static FontCollection fontCollection;
         private static readonly string[] s_imageExtensions = { ".jpg", ".jpeg", ".png", ".webp", ".tga", ".gif", ".bmp" };
@@ -59,6 +63,25 @@ public void SetFontPath(string folder)
             }
         }
 
+        /// <summary>
+        /// Get the perceptual hash
+        /// </summary>
+        /// <param name="path"></param>
+        /// <returns>A hex string representing the hash</returns>
+        public string GetPerceptualHash(string path)
+        {
+            // PerceptualHash, DifferenceHash, AverageHash
+            var hashAlgorithm = new PerceptualHash();
+
+            using var stream = File.OpenRead(path);
+
+            ulong imageHash = hashAlgorithm.Hash(stream);
+
+            var binaryString = Convert.ToString((long)imageHash, 16);
+
+            return binaryString;
+        }
+
         /// <summary>
         /// Create an SHA1 hash from the image data (pixels only) to allow us to find
         /// duplicate images. Note that this ignores EXIF metadata, so the hash will
@@ -194,13 +217,13 @@ public async Task GetCroppedFile( FileInfo source, int x, int y, int width, int
         /// <param name="input"></param>
         /// <param name="output"></param>
         /// <param name="waterMarkText"></param>
-        public void TransformDownloadImage(string input, Stream output, ExportConfig config)
+        public void TransformDownloadImage(string input, Stream output, IExportSettings config)
         {
             Logging.Log($" Running image transform for Watermark: {config.WatermarkText}");
 
             using var img = Image.Load(input, out IImageFormat fmt);
 
-            if (config.Size != Services.ExportSize.FullRes)
+            if (config.Size != ExportSize.FullRes)
             {
                 int maxSize = config.MaxImageSize;
 

Damselfly.Core/ImageProcessing/SkiaSharpProcessor.cs → Damselfly.Core.ImageProcessing/SkiaSharpProcessor.cs

@@ -9,6 +9,7 @@
 using System.Security.Cryptography;
 using System.Threading.Tasks;
 using Damselfly.Core.Models;
+using Damselfly.Core.Utils.Images;
 
 namespace Damselfly.Core.ImageProcessing
 {
@@ -364,7 +365,7 @@ private static SKBitmap AutoOrient(SKBitmap original, SKEncodedOrigin origin)
         /// <param name="input"></param>
         /// <param name="output"></param>
         /// <param name="waterMarkText"></param>
-        public void TransformDownloadImage(string input, Stream output, ExportConfig config)
+        public void TransformDownloadImage(string input, Stream output, IExportSettings config)
         {
             using SKImage img = SKImage.FromEncodedData(input);
             using var bitmap = SKBitmap.FromImage(img);

Damselfly.Core.Interfaces/Damselfly.Core.Interfaces.csproj

@@ -3,9 +3,12 @@
     <None Remove="Microsoft.EntityFrameworkCore.Tools" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0-rc.1.21452.10">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0-rc.2.21480.5">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Damselfly.Core.Utils\Damselfly.Core.Utils.csproj" />
+  </ItemGroup>
 </Project>

Damselfly.Core.Interfaces/IExportSettings.cs

@@ -0,0 +1,15 @@
+using Damselfly.Core.Utils.Constants;
+
+namespace Damselfly.Core.Interfaces
+{
+    public interface IExportSettings
+    {
+        ExportType Type { get; set; }
+        ExportSize Size { get; set; }
+
+        bool KeepFolders { get; set; }
+        string WatermarkText { get; set; }
+        int MaxImageSize { get;  }
+    }
+}
+

Damselfly.Core.Interfaces/IHashProvider.cs

@@ -0,0 +1,10 @@
+using System.IO;
+
+namespace Damselfly.Core.Interfaces
+{
+    public interface IHashProvider
+    {
+        string GetPerceptualHash(string path);
+    }
+}
+

Damselfly.Core/Interfaces/IImageProcessor.cs → Damselfly.Core.Interfaces/IImageProcessor.cs

@@ -1,16 +1,17 @@
 using System.IO;
 using System.Collections.Generic;
-using Damselfly.Core.ImageProcessing;
 using System.Threading.Tasks;
-using Damselfly.Core.Models;
+using Damselfly.Core.Utils.Images;
 
 namespace Damselfly.Core.Interfaces
 {
-    public class ImageProcessResult
+    public interface IImageProcessorFactory
     {
-        public bool ThumbsGenerated { get; set; }
-        public string ImageHash { get; set; }
+        IImageProcessor GetProcessor(string fileExtension);
+        IHashProvider GetHashProvider();
+        void SetContentPath(string contentPath);
     }
+
     /// <summary>
     /// Interface representing a generic image processing pipeline. This
     /// allows us to swap out different implementations etc depending on
@@ -19,8 +20,8 @@ public class ImageProcessResult
     public interface IImageProcessor
     {
         Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles );
-        void TransformDownloadImage(string input, Stream output, ExportConfig config);
         Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo destFile);
+        void TransformDownloadImage(string input, Stream output, IExportSettings exportConfig);
         static ICollection<string> SupportedFileExtensions { get; }
     }
 }