Merge branch 'develop'

Accord/Directory.Build.targets

@@ -1,6 +1,6 @@
 <Project>
  <PropertyGroup>
     <WarningLevel>0</WarningLevel>
-   <NoWarn>1701;1702;3245;0003;0436;</NoWarn>
+   <NoWarn>CA1416,MSB3245,SYSLIB0011,IL2026,IL3000,CS0436,MSB3245,SYSLIB0003,SYSLIB0006,SYSLIB0014,CS0108</NoWarn>
  </PropertyGroup>
 </Project>

Damselfly.Core.DbModels/DBAbstractions/BaseModel.cs

@@ -241,7 +241,10 @@ public override int SaveChanges()
         /// </summary>
         /// <param name="contextDesc"></param>
         /// <returns>The number of entities written to the DB</returns>
-        public async Task<int> SaveChangesAsync(string contextDesc)
+        public async Task<int> SaveChangesAsync(string contextDesc,
+                [System.Runtime.CompilerServices.CallerFilePath] string sourceFilePath = "",
+                [System.Runtime.CompilerServices.CallerMemberNameAttribute] string sourceMethod = "",
+                [System.Runtime.CompilerServices.CallerLineNumber] int lineNumber = 0 )
         {
             if (ReadOnly)
             {
@@ -250,30 +253,47 @@ public async Task<int> SaveChangesAsync(string contextDesc)
                 return 1;
             }
 
-            try
+            int retriesRemaining = 3;
+            int recordsWritten = 0;
+
+            while ( retriesRemaining > 0 )
             {
-                // Write to the DB
-                var watch = new Stopwatch("SaveChanges" + contextDesc);
+                try
+                {
+                    // Write to the DB
+                    var watch = new Stopwatch("SaveChanges" + contextDesc);
 
-                LogChangeSummary();
+                    LogChangeSummary();
 
-                int written = await base.SaveChangesAsync();
+                    recordsWritten = await base.SaveChangesAsync();
 
-                Logging.LogTrace("{0} changes written to the DB", written);
+                    Logging.LogTrace("{0} changes written to the DB", recordsWritten);
 
-                watch.Stop();
+                    watch.Stop();
 
-                return written;
-            }
-            catch (Exception ex)
-            {
-                if (ex.InnerException != null)
-                    Logging.Log("Exception - DB WRITE FAILED. InnerException: {0}", ex.InnerException.Message);
-                else
-                    Logging.Log("Exception - DB WRITE FAILED: {0}", ex.Message);
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    if (ex.Message.Contains("database is locked") && retriesRemaining > 0 )
+                    {
+                        Logging.LogWarning($"Database locked for {contextDesc} - sleeping for 5s and retying {retriesRemaining}...");
+                        retriesRemaining--;
+                        await Task.Delay(5 * 1000);
+                    }
+                    else
+                    {
+                        Logging.LogError($"Exception - DB WRITE FAILED for {contextDesc}: {ex.Message}" );
+                        Logging.LogError($"  Called from {sourceMethod} line {lineNumber} in {sourceFilePath}.");
+                        if (ex.InnerException != null)
+                            Logging.LogError("  Exception - DB WRITE FAILED. InnerException: {0}", ex.InnerException.Message);
+
+                    }
+                }
 
-                return 0;
             }
+
+            return recordsWritten;
         }
 
         /// <summary>

Damselfly.Core.Utils/Constants/ConfigSettings.cs

@@ -16,6 +16,7 @@ public class ConfigSettings
 
         public const string ImportSidecarKeywords = "ImportSidecarKeywords";
 
+        public const string AIProcessingTimeRange = "AIProcessingTimeRange";
         public const string AzureEndpoint = "AzureEndpoint";
         public const string AzureApiKey = "AzureApiKey";
         public const string AzureDetectionType = "AzureDetectionType";

Damselfly.Core.Utils/Utils/DateTimeExtensions.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace Damselfly.Core.Utils
+{
+    public static class DateTimeExtensions
+    {
+        public static TimeSpan? LocalTimeSpanToUTC(this TimeSpan? ts)
+        {
+            if (ts.HasValue)
+            {
+                DateTime dt = DateTime.Now.Date.Add(ts.Value);
+                DateTime dtUtc = dt.ToUniversalTime();
+                TimeSpan tsUtc = dtUtc.TimeOfDay;
+                return tsUtc;
+            }
+
+            return null;
+        }
+
+        public static TimeSpan? UTCTimeSpanToLocal(this TimeSpan? tsUtc)
+        {
+            if (tsUtc.HasValue)
+            {
+                DateTime dtUtc = DateTime.UtcNow.Date.Add(tsUtc.Value);
+                DateTime dt = dtUtc.ToLocalTime();
+                TimeSpan ts = dt.TimeOfDay;
+                return ts;
+            }
+
+            return null;
+        }
+    }
+}

Damselfly.Core.Utils/Utils/Logging.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using System.Threading;
 using Serilog;
 using Serilog.Core;
@@ -45,34 +46,44 @@ public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory)
         /// <returns></returns>
         public static Logger InitLogs()
         {
-            if (!Directory.Exists(LogFolder))
-                Directory.CreateDirectory(LogFolder);
-
-            logLevel.MinimumLevel = LogEventLevel.Information;
-
-            if (Verbose)
-                logLevel.MinimumLevel = LogEventLevel.Verbose;
-
-            if (Trace)
-                logLevel.MinimumLevel = LogEventLevel.Debug;
-
-            string logFilePattern = Path.Combine(LogFolder, "Damselfly-.log");
-
-            logger = new LoggerConfiguration()
-                .MinimumLevel.ControlledBy(logLevel)
-                .Enrich.With( new ThreadIDEnricher() )
-                .WriteTo.Console(outputTemplate: template)
-                .WriteTo.File( logFilePattern,
-                               outputTemplate: template,
-                               rollingInterval: RollingInterval.Day,
-                               fileSizeLimitBytes:104857600,
-                               retainedFileCountLimit:10)
-                .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
-                .CreateLogger();
-
-            logger.Information("=== Damselfly Log Started ===");
-            logger.Information("Log folder: {0}", LogFolder);
-            logger.Information("LogLevel: {0}", logLevel.MinimumLevel);
+            try
+            {
+                if (!Directory.Exists(LogFolder))
+                {
+                    Console.WriteLine($"Creating log folder {LogFolder}");
+                    Directory.CreateDirectory(LogFolder);
+                }
+
+                logLevel.MinimumLevel = LogEventLevel.Information;
+
+                if (Verbose)
+                    logLevel.MinimumLevel = LogEventLevel.Verbose;
+
+                if (Trace)
+                    logLevel.MinimumLevel = LogEventLevel.Debug;
+
+                string logFilePattern = Path.Combine(LogFolder, "Damselfly-.log");
+
+                logger = new LoggerConfiguration()
+                    .MinimumLevel.ControlledBy(logLevel)
+                    .Enrich.With(new ThreadIDEnricher())
+                    .WriteTo.Console(outputTemplate: template)
+                    .WriteTo.File(logFilePattern,
+                                   outputTemplate: template,
+                                   rollingInterval: RollingInterval.Day,
+                                   fileSizeLimitBytes: 104857600,
+                                   retainedFileCountLimit: 10)
+                    .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
+                    .CreateLogger();
+
+                logger.Information("=== Damselfly Log Started ===");
+                logger.Information("Log folder: {0}", LogFolder);
+                logger.Information("LogLevel: {0}", logLevel.MinimumLevel);
+            }
+            catch( Exception ex )
+            {
+                Console.WriteLine($"Unable to initialise logs: {ex}");
+            }
 
             return logger;
         }

Damselfly.Core/ImageProcessing/ImageMagickProcessor.cs

@@ -49,7 +49,7 @@ private void CheckToolStatus()
         /// </summary>
         /// <param name="source">Source.</param>
         /// <param name="sizes">Sizes.</param>
-        public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles )
+        public async Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles )
         {
             // This processor doesn't support hash creation
             ImageProcessResult result = new ImageProcessResult { ThumbsGenerated = false, ImageHash = string.Empty };
@@ -119,7 +119,7 @@ public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileIn
                     {
                         process.BeginErrorReadLine();
                         process.BeginOutputReadLine();
-                        process.WaitForExit();
+                        await process.WaitForExitAsync();
 
                         if (process.ExitCode == 0)
                         {
@@ -138,7 +138,7 @@ public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileIn
             else
                 Logging.LogVerbose("Thumbs already exist in all resolutions. Skipping...");
 
-            return Task.FromResult(result);
+            return result;
         }
 
         private static void Process_OutputDataReceived(object sender, DataReceivedEventArgs e)
@@ -154,5 +154,10 @@ public void TransformDownloadImage(string input, Stream output, ExportConfig con
         {
             throw new NotImplementedException();
         }
+
+        public async Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo destFile)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

Damselfly.Core/ImageProcessing/ImageSharpProcessor.cs

@@ -95,6 +95,12 @@ public static string GetHash(Image<Rgba32> image)
             return result;
         }
 
+        /// <summary>
+        /// Draw rectangles onto a file
+        /// </summary>
+        /// <param name="path"></param>
+        /// <param name="rects"></param>
+        /// <param name="output"></param>
         public static void DrawRects( string path, List<Rectangle> rects, string output )
         {
             using var image = Image.Load<Rgba32>(path);
@@ -115,14 +121,14 @@ public static void DrawRects( string path, List<Rectangle> rects, string output
         /// </summary>
         /// <param name="source"></param>
         /// <param name="destFiles"></param>
-        public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles)
+        public async Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles)
         {
             var result = new ImageProcessResult();
             Stopwatch load = new Stopwatch("ImageSharpLoad");
 
             // Image.Load(string path) is a shortcut for our default type. 
             // Other pixel formats use Image.Load<TPixel>(string path))
-            using var image = Image.Load<Rgba32>(source.FullName);
+            using var image = await Image.LoadAsync<Rgba32>(source.FullName);
 
             // We've got the image in memory. Create the hash. 
             result.ImageHash = GetHash(image);
@@ -160,14 +166,26 @@ public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileIn
                 // So we always resize the previous image, which will be faster for each iteration
                 // because each previous image is progressively smaller. 
                 image.Mutate(x => x.Resize(opts));
-                image.Save(dest.FullName);
+                await image.SaveAsync(dest.FullName);
 
                 result.ThumbsGenerated = true;
             }
 
             thumbs.Stop();
 
-            return Task.FromResult(result);
+            return result;
+        }
+
+        public async Task GetCroppedFile( FileInfo source, int x, int y, int width, int height, FileInfo destFile )
+        {
+            // Image.Load(string path) is a shortcut for our default type. 
+            // Other pixel formats use Image.Load<TPixel>(string path))
+            using var image = Image.Load<Rgba32>(source.FullName);
+
+            var rect = new Rectangle(x, y, width, height);
+            image.Mutate(x => x.AutoOrient());
+            image.Mutate(x => x.Crop( rect ));
+            await image.SaveAsync(destFile.FullName);
         }
 
         /// <summary>

Damselfly.Core/ImageProcessing/SkiaSharpProcessor.cs

@@ -139,6 +139,39 @@ public Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileIn
             return Task.FromResult(result);
         }
 
+        /// <summary>
+        /// Crop a file
+        /// </summary>
+        /// <param name="source"></param>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="width"></param>
+        /// <param name="height"></param>
+        /// <param name="dest"></param>
+        /// <returns></returns>
+        public Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo dest)
+        {
+            try
+            {
+                using var sourceBitmap = LoadOrientedBitmap(source, width);
+
+                // setup crop rect
+                var cropRect = new SKRectI(x, y, x + width, y + height);
+                var cropped = Crop(sourceBitmap, cropRect);
+                using SKData data = cropped.Encode(SKEncodedImageFormat.Jpeg, 90);
+
+                using (var stream = new FileStream(dest.FullName, FileMode.Create, FileAccess.Write))
+                    data.SaveTo(stream);
+            }
+            catch (Exception ex)
+            {
+                Logging.Log($"Exception during Crop processing: {ex.Message}");
+                throw;
+            }
+
+            return Task.CompletedTask;
+        }
+
         /// <summary>
         /// Crop the image to fit within the dimensions specified. 
         /// </summary>
@@ -167,15 +200,26 @@ private SKBitmap Crop(SKBitmap original, SKSize maxSize)
                     Bottom = original.Height - cropTopBottom + cropTopBottom / 2
                 };
 
-                // crop
-                SKBitmap bitmap = new SKBitmap(cropRect.Width, cropRect.Height);
-                original.ExtractSubset(bitmap, cropRect);
-                return bitmap;
+                return Crop(original, cropRect);
             }
             else
                 return original.Copy();
         }
 
+        /// <summary>
+        /// Crop to a rectangle
+        /// </summary>
+        /// <param name="original"></param>
+        /// <param name="cropRect"></param>
+        /// <returns></returns>
+        private SKBitmap Crop( SKBitmap original, SKRectI cropRect)
+        {
+            // crop
+            SKBitmap bitmap = new SKBitmap(cropRect.Width, cropRect.Height);
+            original.ExtractSubset(bitmap, cropRect);
+            return bitmap;
+        }
+
         /// <summary>
         /// Loads an image from a disk file, decoding for the optimal required
         /// size so that we don't load the entire image for a smaller target,

Damselfly.Core/Interfaces/IImageProcessor.cs

@@ -20,6 +20,7 @@ public interface IImageProcessor
     {
         Task<ImageProcessResult> CreateThumbs(FileInfo source, IDictionary<FileInfo, ThumbConfig> destFiles );
         void TransformDownloadImage(string input, Stream output, ExportConfig config);
+        Task GetCroppedFile(FileInfo source, int x, int y, int width, int height, FileInfo destFile);
         static ICollection<string> SupportedFileExtensions { get; }
     }
 }